diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..66e239f --- /dev/null +++ b/Readme.md @@ -0,0 +1,57 @@ +qmongoid +======== + +Generate unique ids quickly. The ids are constructed like MongoDB +document ids, built out of a timestamp, system id, process id and sequence +number. Similar to require('bson').ObjectID(), but 10x faster (1.8 million/sec). + +The ids are guaranteed unique on any one server, and can be configured +to be unique across a cluster of up to 16 million (2^24) servers. +Uniqueness is guaranteed by unique {server, process} id pairs. + +## Functions + +### mongoid( ) + +generates ids that are unique to this server. The ids are generated by a +MongoId singleton initialized with a random machine id. All subsequent calls +to mongoid() in this process will fetch ids from this singleton. + + // ids with a randomly chosen system id (here 0x40e281) + var mongoid = require('mongoid-js'); + var id1 = mongoid(); // => 543f376340e2816497000001 + var id2 = mongoid(); // => 543f376340e2816497000002 + +### new MongoId( systemId ).fetch( ) + +unique id factory that embeds the given system id in each generated unique id. +By a systematic assignment of system ids to servers, this approach can guarantee +globally unique ids (ie, globally for an installation). + + // ids with a unique system id + var MongoId = require('mongoid-js').MongoId; + var systemId = 4656; + var idFactory = new MongoId(systemId); + var id1 = idFactory().fetch(); // => 543f3789001230649f000001 + var id2 = idFactory().fetch(); // => 543f3789001230649f000002 + +### MongoId.parse( idString ) + +Decompose the id string into its parts -- unix timestamp, machine id, +process id and sequence number. Unix timestamps are seconds since the +start of the epoch (1970-01-01 UTC) + + var parts = MongoId.parse("543f376340e2816497000013"); + // => { timestamp: 1413429091, + // machineid: 4252289, + // pid: 25751, + // sequence: 19 } + +### MongoId.getTimestamp( idString ) + +Return just the javascript timestamp part of the id. Javascript timestamps +are milliseconds since the start of the epoch (they are 1000 x more than the +unix timestamp.) + + MongoId.getTimestamp("543f376340e2816497000013"); + // => 1413429091000 diff --git a/index.js b/index.js new file mode 100644 index 0000000..07560d3 --- /dev/null +++ b/index.js @@ -0,0 +1 @@ +module.exports = require('./mongoid.js'); diff --git a/mongoid.js b/mongoid.js new file mode 100644 index 0000000..f549f2a --- /dev/null +++ b/mongoid.js @@ -0,0 +1,108 @@ +/** + * Generate unique ids in the style of mongodb. + * Ids are a hex number built out of the timestamp, a per-server unique id, + * the process id and a sequence number. + * + * Copyright (C) 2014 Andras Radics + * Licensed under the Apache License, Version 2.0 + * + * MongoDB object ids are 12 bytes (24 hexadecimal chars), composed out of + * a Unix timestamp (seconds since the epoch), a system id, the process id, + * and a monotonically increasing sequence number. + * The Unix epoch is 1970-01-01 00:00:00 GMT. + * + * timestamp 4B (8 hex digits) + * machine id 3B (6 digits) + * process id 2B (4 digits) + * sequence 3B (6 digits) + */ + + +'use strict'; + +module.exports = mongoid; +module.exports.mongoid = MongoId; +module.exports.MongoId = MongoId; + +var globalSingleton = null; + +function mongoid( ) { + if (!globalSingleton) globalSingleton = new MongoId(); + return globalSingleton.fetch(); +} + +function MongoId( machineId ) { + // if called as a function, return an id from the singleton + if (this === global || !this) return mongoid(); + + // if no machine id specified, use a 3-byte random number + if (!machineId) machineId = Math.floor(Math.random() * 0x1000000); + else if (machineId < 0 || machineId > 0x1000000) + throw new Error("machine id out of range 0.." + parseInt(0x1000000)); + + this.machineIdStr = hexFormat(machineId, 6); + this.pidStr = hexFormat(process.pid, 4); + this.lastTimestamp = null; + this.sequenceId = 0; + this.id = null; +} + +MongoId.prototype.fetch = function() { + var id; + var timestamp = Math.floor(Date.now()/1000); + + // soft-init on first call and on every new second + if (timestamp !== this.lastTimestamp) { + this.lastTimestamp = timestamp; + this.timestampStr = hexFormat(timestamp, 8); + if (!this.sequenceId) this.sequenceStartTimestamp = timestamp; + } + + // sequence wrapping and overflow check + if (this.sequenceId >= 0x1000000) { + if (timestamp === this.sequenceStartTimestamp) { + throw new Error("mongoid sequence overflow: more than 16 million ids generated in 1 second"); + } + this.sequenceId = 0; + this.sequenceStartTimestamp = timestamp; + } + + id = this.timestampStr + this.machineIdStr + this.pidStr + hexFormat(++this.sequenceId, 6); + return id; +}; + +function hexFormat(n, width) { + var s = n.toString(16); + while (s.length + 2 < width) s = "00" + s; + while (s.length < width) s = "0" + s; + return s; +} +MongoId.prototype.hexFormat = hexFormat; + +// each MongoId object also evaluates to a per-object id string +MongoId.prototype.toString = function( ) { + return this.id ? this.id : this.id = this.fetch(); +}; + +MongoId.parse = function( idstring ) { + if (typeof idstring !== 'string') idstring = "" + idstring; + return { + timestamp: parseInt(idstring.slice( 0, 0+8), 16), + machineid: parseInt(idstring.slice( 8, 8+6), 16), + pid: parseInt(idstring.slice(14, 14+4), 16), + sequence: parseInt(idstring.slice(18, 18+6), 16) + }; +}; +MongoId.prototype.parse = function( idstring ) { + return MongoId.parse(this.toString()); +}; + +// return the javascript timestamp (milliseconds) embedded in the id. +// Note that the ids embed unix timestamps (seconds precision). +MongoId.getTimestamp = function( idstring ) { + return parseInt(idstring.slice(0, 8), 16) * 1000; +}; +MongoId.prototype.getTimestamp = function( idstring ) { + return MongoId.getTimestamp(this.toString()); +}; + diff --git a/package.json b/package.json new file mode 100644 index 0000000..bf5b9a4 --- /dev/null +++ b/package.json @@ -0,0 +1,30 @@ +{ + "name": "mongoid-js", + "version": "1.0.0", + "description": "quick unique ids, mongoid compatible", + "license": "Apache-2.0", + "main": "index.js", + "author": { + "name": "Andras", + "url": "http://github.com/andrasq" + }, + "engines": { + "node": ">=0.0.0" + }, + "scripts": { + "test": "nodeunit" + }, + "keywords": [ + "Andras", + "quick", + "mongoid", + "globally", + "unique", + "ids" + ], + "dependencies": { + }, + "devDependencies": { + "nodeunit": "0.9.0" + } +} diff --git a/test/test-mongoid.js b/test/test-mongoid.js new file mode 100644 index 0000000..d58aa4a --- /dev/null +++ b/test/test-mongoid.js @@ -0,0 +1,152 @@ +'use stricf'; + +var mongoid = require('../mongoid'); +var MongoId = require('../mongoid').MongoId; + +function uniqid() { + return Math.floor(Math.random() * 0x1000000); +} + +function testUnique( test, a ) { + var ids = {} + for (var i in a) { + var v = a[i]; + if (ids[v] !== undefined) test.fail("index " + i + ": duplicate id " + v + ", seen at index " + ids[v]); + ids[v] = i; + } +} + +module.exports.require = { + setUp: function(cb) { + cb(); + }, + + tearDown: function(cb) { + cb(); + }, + + tests: { + testShouldExportMongoidFunction: function(test) { + var mongoid = require('../mongoid'); + test.ok(mongoid.mongoid); + var id = mongoid(); + test.equal(id.length, 24); + test.done(); + }, + + testShouldExportMongoIdClass: function(test) { + var MongoId = require('../mongoid'); + test.ok(MongoId.MongoId); + test.done(); + }, + + testShouldBeUsableAsFunction: function(test) { + var mongoid = require('../mongoid'); + test.ok(typeof mongoid === 'function'); + test.ok(typeof mongoid() === 'string'); + test.done(); + }, + }, +}; + +module.exports.mongoid_function = { + testShouldReturn24CharHexString: function(test) { + var id = mongoid(); + test.ok(id.match(/[0-9a-fA-F]{24}/), "should return a 24-char id string"); + test.done(); + }, + + testShouldReturnUniqueIds: function(test) { + var ids = []; + for (var i=0; i<20000; i++) ids.push(mongoid()); + testUnique(test, ids); + test.done(); + }, + + testMongoidSpeed: function(test) { + var t1 = Date.now(); + for (var i=0; i<10000; i++) mongoid(); + var t2 = Date.now(); + //console.log("mongoid(): 10k in " + (t2-t1) + " ms"); + test.ok(t2-t1 < 100, "should generate > 100k ids / sec"); + test.done(); + }, +}; + +module.exports.MongoId_class = { + testShouldReturnObject: function(test) { + var obj = new MongoId(0x123); + test.ok(typeof obj == 'object'); + test.done(); + }, + + testShouldHaveHexFormatMethod: function(test) { + test.ok(typeof (new MongoId()).hexFormat == 'function'); + test.done(); + }, + + testSameObjectShouldReturnSameIdString: function(test) { + var obj = new MongoId(0x1234); + var id1 = "" + obj; + var id2 = "" + obj; + test.equal(id1, id2); + test.done(); + }, + + testShouldUseConstructorMachineId: function(test) { + var hexFormat = MongoId.prototype.hexFormat; + var machineid = uniqid(); + var obj = new MongoId(machineid); + var id = obj.fetch(); + test.equal(id.slice(8, 8+6), hexFormat(machineid, 6), "id " + id + " should contain machineid " + machineid.toString(16)); + test.done(); + }, + + testShouldParseId: function(test) { + var timestamp = Math.floor(Date.now()/1000); + var obj = new MongoId(0x123456); + var hash = obj.parse(obj.toString()); + test.equal(hash.machineid, 0x123456); + test.equal(hash.sequence, 1); + test.ok(hash.timestamp === timestamp || hash.timestamp === timestamp+1); + test.equal(hash.pid, process.pid); + test.done(); + }, + + testIdShouldContainParsedParts: function(test) { + var obj = new MongoId(); + var hexFormat = obj.hexFormat; + var id = obj.toString(); + var hash = obj.parse(id); + var id2 = hexFormat(hash.timestamp, 8) + + hexFormat(hash.machineid, 6) + + hexFormat(hash.pid, 4) + + hexFormat(hash.sequence, 6); + test.equal(id, id2); + test.done(); + }, + + testShouldGetTimestamp: function(test) { + var obj = new MongoId(); + var id = mongoid(); + var parts = obj.parse(id); + var timestamp = obj.getTimestamp(id); + test.equal(timestamp, parts.timestamp * 1000); + test.done(); + }, + + testUniqueObjectsShouldReturnUniqueIds: function(test) { + var ids = []; + for (var i=0; i<20000; i++) ids.push((new MongoId(i)).toString()); + testUnique(test, ids); + test.done(); + }, + + testUniqueObjectsShouldReturnUniqueIds: function(test) { + var ids = []; + var obj = new MongoId(0x12345); + for (var i=0; i<20000; i++) ids.push(obj.fetch()); + testUnique(test, ids); + test.done(); + }, +}