From 89813cbd28571e72185e974184c068002423807d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sat, 28 Feb 2015 20:50:30 +0000 Subject: [PATCH 001/304] Allow a TLS option --- lib/connection.js | 18 ++++-- test/certs/my-private-root-ca.crt.pem | 22 +++++++ test/certs/my-private-root-ca.key.pem | 27 ++++++++ test/certs/my-private-root-ca.srl | 1 + test/certs/my-server.crt.pem | 20 ++++++ test/certs/my-server.csr.pem | 17 +++++ test/certs/my-server.key.pem | 27 ++++++++ test/tls.js | 92 +++++++++++++++++++++++++++ 8 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 test/certs/my-private-root-ca.crt.pem create mode 100644 test/certs/my-private-root-ca.key.pem create mode 100644 test/certs/my-private-root-ca.srl create mode 100644 test/certs/my-server.crt.pem create mode 100644 test/certs/my-server.csr.pem create mode 100644 test/certs/my-server.key.pem create mode 100644 test/tls.js diff --git a/lib/connection.js b/lib/connection.js index 3dd46d4..22cfa14 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -1,4 +1,5 @@ var net = require('net'); +var tls = require('tls'); var Promise = require('bluebird'); var events = require("events"); var util = require('util'); @@ -37,10 +38,19 @@ function Connection(r, options, resolve, reject) { this.open = false; // true only if the user can write on the socket this.timeout = null; - self.connection = net.connect({ - host: self.host, - port: self.port - }); + this.tls = options.tls || false; + if (!this.tls) { + self.connection = net.connect({ + host: self.host, + port: self.port + }); + } else { + if (this.tls === true) { + self.connection = tls.connect(this.port, this.host); + } else { + self.connection = tls.connect(this.port, this.host, this.tls); + } + } self.timeoutOpen = setTimeout(function() { self.connection.end(); // Send a FIN packet diff --git a/test/certs/my-private-root-ca.crt.pem b/test/certs/my-private-root-ca.crt.pem new file mode 100644 index 0000000..df2702b --- /dev/null +++ b/test/certs/my-private-root-ca.crt.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDoTCCAomgAwIBAgIJAPvM7fEPUQ4AMA0GCSqGSIb3DQEBCwUAMGcxCzAJBgNV +BAYTAlVTMQ0wCwYDVQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEjMCEGA1UECgwa +QUNNRSBTaWduaW5nIEF1dGhvcml0eSBJbmMxFDASBgNVBAMMC2V4YW1wbGUuY29t +MB4XDTE1MDIyODIwNDAzM1oXDTE3MTIxODIwNDAzM1owZzELMAkGA1UEBhMCVVMx +DTALBgNVBAgMBFV0YWgxDjAMBgNVBAcMBVByb3ZvMSMwIQYDVQQKDBpBQ01FIFNp +Z25pbmcgQXV0aG9yaXR5IEluYzEUMBIGA1UEAwwLZXhhbXBsZS5jb20wggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDHPDZhUnvrpGIeBke2R9IBLPGKJiPj +PX8MWCLiNodwhovlcyQ8jE8/P+/9pNU0K6D0BoFfzOz8YaUinoSGkB2PI5r1lcjS +Av8nzZD8RvFxSYHxIMV8oFt1rjA/DoUngUksguDKSWIQM3Qfv93GydZBO8GP6FeW +RPnsaJZBuKT/BWGb4XVFWq7yFnjhQ4RWh7o2Q6EIh89BJX7ZzxdY9rnB1Y0MmIQv +9ZYs9kMKfcz2wWRtinxLX0HtrnNVaN7KKKDMdiMCR0f+oaagZ+GN4wQF2SxEiTYR +mZZ2Y7tMZOdN5zr3PGs7ftYpKwngqvtaSxh97mdO8B6FvoHSwrzrI9JTAgMBAAGj +UDBOMB0GA1UdDgQWBBTzvtsbiPS9NEOs8rEn2qZlUiIwPjAfBgNVHSMEGDAWgBTz +vtsbiPS9NEOs8rEn2qZlUiIwPjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQCfthEqAp21dp4xPpux7/M/029h6wq7cvRexzF5fEzZT2P0+p5gNCZwtk34 +Hrx7x2RTQIsA6qc4bz5Dbl9nv1u1v1iOixNfdtEYqUfV/AF59hfvea6fXqa1hRt2 +cgA5X3DdYvwD3Uc8j/F44b08cLYVIjZCieSSXlVyN136Vz7ZXjXSA9JXQXOdpeI7 +7jEXSgR0zfUY+x3JaUMxooE7wEUHo97wiMS73ZM8J0vkZchrGmHZGccl4v9daThF +j4hXlg1dSjn6TGCTinMiio6YqxZLEs+OtK7mW2oAeev5b6JNC9Ohz6m6uO8Rdg0c +wVK9u7SzxggilfK9Hrw9envb86Em +-----END CERTIFICATE----- diff --git a/test/certs/my-private-root-ca.key.pem b/test/certs/my-private-root-ca.key.pem new file mode 100644 index 0000000..89a31b6 --- /dev/null +++ b/test/certs/my-private-root-ca.key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAxzw2YVJ766RiHgZHtkfSASzxiiYj4z1/DFgi4jaHcIaL5XMk +PIxPPz/v/aTVNCug9AaBX8zs/GGlIp6EhpAdjyOa9ZXI0gL/J82Q/EbxcUmB8SDF +fKBbda4wPw6FJ4FJLILgykliEDN0H7/dxsnWQTvBj+hXlkT57GiWQbik/wVhm+F1 +RVqu8hZ44UOEVoe6NkOhCIfPQSV+2c8XWPa5wdWNDJiEL/WWLPZDCn3M9sFkbYp8 +S19B7a5zVWjeyiigzHYjAkdH/qGmoGfhjeMEBdksRIk2EZmWdmO7TGTnTec69zxr +O37WKSsJ4Kr7WksYfe5nTvAehb6B0sK86yPSUwIDAQABAoIBACHamWLwIR5pUELJ +zmWqzoRknqZa1L5INM5kK4lEur7kHeFfL0kajlyxAJaY66FxyGeus00UBTDdZrH7 +PYmf82XfJvWT/guxdqnMxFYZt+0IEIxohSYoGWJlts0AKqAL1+M4WfoFfiRCmfjL +IWQl58It5TYOxNND5TPGkcGZHnA2rNa+9SV8ILELCsTYL8dKI07WEGiFqsmrGTfv +O+8QlGtE2i97ze1+8tcQ0XbqM8SAncq2LIq5NHmulds0fBnm4Z3JI0YpaZXS60xu +AA040AJNWnkIg3j+MFyM48tWuhm5QnqZc+rg9tJKo9UaTsZeGVGTUS3k6uOuknIl +/Px5hAECgYEA9hPFC3SYBD5jWZGE6YOT61rAlDV4MypZJfG9lgP2qSvL2ceSjy3i +4dgU80rpz5DWYrkifGOL8D/JIiGA0SmKp716hmA4Iq5NlyA9jAwGzJ7Fv2133a6V +NujXzw7gd0tyjIP1u+EhSTpAOSqt/a4rCn/J0enI+LgWJ+JjOMRQlCcCgYEAz0Tl +tAMfNueo63O4+zQyF9i4xMPXQf1Imk7c0Y4XhmH2cs1YaEned4DvF6mb1hJSrZUM ++3ic5pQUCltkky6GiSKZA0UT7uUBpNlUtrh2BvaVtMkFp8zo8JfomsOHQUld9tJE +JPRePz/NnfVLnUn9fLZ/AMUDRICRCb0Ng1wCT/UCgYA9N83de7QV2i99KOaX8VPr +STh/po2wrOhguDMiDvpeO7cLBss/M3suFEOiAxEHwlCTXttldp5ptnSjORKC9oK4 +GDtTErzkY9iJsEufe551aUpoxSL802HrtyF5MYpGI07sBigsBWXygbtYfXWrlpi+ +k+vxfOeyeJmKcpecl7o0IQKBgG7ab2wOe2RXFA4Rj26y5NPCMlfyUi9V8r2szgc5 +0rUpsbCGyPDGCTiq+bUTHmio1hVGcXIReQENU4aSiMD7EjNIEMQh7t5HFtD+94Le +NynUOg89sEulmTQMWvhi+PTe82Vt4cpN9BYp3qPLrIxMJUnNJTHZLL03cdrehNqr +nAT9AoGBAL1pa0z6Edhy539mKGBDdlWpDseakyD4hMwlSf633s8bbOtGNiU4lON3 +yc3eCqtsDFy1Ml/QyYsh7zE9omR8+2H841ubEXOaocEXYlg0lqQOgsowXg73C/sy +cDpEl84WgUufXJLFL8m7TnjFUTpIU+bIU60UtzQYmhJDQfGBMTfD +-----END RSA PRIVATE KEY----- diff --git a/test/certs/my-private-root-ca.srl b/test/certs/my-private-root-ca.srl new file mode 100644 index 0000000..7d09bb5 --- /dev/null +++ b/test/certs/my-private-root-ca.srl @@ -0,0 +1 @@ +88E48FE6D1A917DB diff --git a/test/certs/my-server.crt.pem b/test/certs/my-server.crt.pem new file mode 100644 index 0000000..ed2c033 --- /dev/null +++ b/test/certs/my-server.crt.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDOzCCAiMCCQCI5I/m0akX2zANBgkqhkiG9w0BAQsFADBnMQswCQYDVQQGEwJV +UzENMAsGA1UECAwEVXRhaDEOMAwGA1UEBwwFUHJvdm8xIzAhBgNVBAoMGkFDTUUg +U2lnbmluZyBBdXRob3JpdHkgSW5jMRQwEgYDVQQDDAtleGFtcGxlLmNvbTAeFw0x +NTAyMjgyMDQwMzRaFw0xNjA3MTIyMDQwMzRaMFgxCzAJBgNVBAYTAlVTMQ0wCwYD +VQQIDARVdGFoMQ4wDAYDVQQHDAVQcm92bzEWMBQGA1UECgwNQUNNRSBUZWNoIElu +YzESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAtYAhzTsKj0nN5HsHptn0eJ4yi9UKi0h9qnzRoxtEVPy1hAfruGCQjz6F +A29CNUP3wPEuiECCH+cWiVR6c5WdrRS0dHX73MT7ow/TaINIS8QXCkTXgMQxc6J7 ++3DZj8QliuqxRDjhOc99Xi9PgIBXZHWMPHxyh1iqjST5X0WQRpLz7ixGH9dtPRCL +UKpDlPA6lAUoCkF5yim/M8oeDFySmFVFKSioYJDL7+hcbAzSy513MP9krTfBtVDz +LQufsjwoxZr4OowovuiZZPmuSsoleJtyvVx37/MGGvYGU2CZ13cMqfckfn8el77W +FmqPEHazW8yAkk25o+l+w668QAO/0QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA9 +TELj0GS/G/dvuWrTYPDopS4/fmFDv92dyPh6AuN33ok0TizCPkVyhUTO6pcalUZU +sC2SdPovsxZx9eUNNkkIleYCHem4f50QkyYKewEHAQC86dMEm3sEsvCVsdkJ+pop +HDmxtiL6qEihWn5q3lyCYywfBgKUipDxkZ9+0hRJ8E7ak9p6MsGDq0QJtL5EV80s +vnFT+pT0UyR2qxKibc7M/Muu+G3GYJR2zppjz8+0Haq5NTtO05Uu9fiGypKJpiRT +EWLEtbylSdWV2QIbzjsWtAjAA/nIyIzoP5o8CQVUJ+MCkX8STkSkHvt5CgRHqFl9 +r4BTwYUFDy+BfPFfdwd/ +-----END CERTIFICATE----- diff --git a/test/certs/my-server.csr.pem b/test/certs/my-server.csr.pem new file mode 100644 index 0000000..61f5b60 --- /dev/null +++ b/test/certs/my-server.csr.pem @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICnTCCAYUCAQAwWDELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFV0YWgxDjAMBgNV +BAcMBVByb3ZvMRYwFAYDVQQKDA1BQ01FIFRlY2ggSW5jMRIwEAYDVQQDDAlsb2Nh +bGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1gCHNOwqPSc3k +ewem2fR4njKL1QqLSH2qfNGjG0RU/LWEB+u4YJCPPoUDb0I1Q/fA8S6IQIIf5xaJ +VHpzlZ2tFLR0dfvcxPujD9Nog0hLxBcKRNeAxDFzonv7cNmPxCWK6rFEOOE5z31e +L0+AgFdkdYw8fHKHWKqNJPlfRZBGkvPuLEYf1209EItQqkOU8DqUBSgKQXnKKb8z +yh4MXJKYVUUpKKhgkMvv6FxsDNLLnXcw/2StN8G1UPMtC5+yPCjFmvg6jCi+6Jlk ++a5KyiV4m3K9XHfv8wYa9gZTYJnXdwyp9yR+fx6XvtYWao8QdrNbzICSTbmj6X7D +rrxAA7/RAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAGOt0xMNt7HB9QWMw0A9Q +bJgazLmJunvO5hpV+77cmyKs29w1DVGhEzE6fjporngkVz1Flso0NFYVK2oLhgfu +Q8QHEgm1a1T7hHuiPFk6sLNS/Ahq565O3VXzRp34OKhino7N9AUc2XXeFf8UF+jb +KRJtqCqRvqO/Ukzc579W6fOqGpP/ojoG8tOsDFg2JKrLT5rp3CM9vYVFpirAg/EV +dRKIZHReV6Hh0VM3Be1QCFtVeFvPvtm5BJ6DYvJpr7k4MqdCR6nZ3u652BinU1BB +E/NTSL1As0gHTk9r475ExXFQ+meABunVmO83ppo7sp42mZHFXzqFbUAN9hdp59o3 +HA== +-----END CERTIFICATE REQUEST----- diff --git a/test/certs/my-server.key.pem b/test/certs/my-server.key.pem new file mode 100644 index 0000000..da600e1 --- /dev/null +++ b/test/certs/my-server.key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAtYAhzTsKj0nN5HsHptn0eJ4yi9UKi0h9qnzRoxtEVPy1hAfr +uGCQjz6FA29CNUP3wPEuiECCH+cWiVR6c5WdrRS0dHX73MT7ow/TaINIS8QXCkTX +gMQxc6J7+3DZj8QliuqxRDjhOc99Xi9PgIBXZHWMPHxyh1iqjST5X0WQRpLz7ixG +H9dtPRCLUKpDlPA6lAUoCkF5yim/M8oeDFySmFVFKSioYJDL7+hcbAzSy513MP9k +rTfBtVDzLQufsjwoxZr4OowovuiZZPmuSsoleJtyvVx37/MGGvYGU2CZ13cMqfck +fn8el77WFmqPEHazW8yAkk25o+l+w668QAO/0QIDAQABAoIBAQCTosdcBoOfbHxz +NvRRb4chuiUJg2DRTUUTMfbggySBMRKBdGP0lW/hZQyTb6PeagwUwNqF8FsokjPN +v47fiDs699WVygb2XLEHwa7Im8lp//Zx3u+hCXXkRq4tKG2AOSAEsMVC5jWmbbXf +sAIuV7c+uzM4XE5Y8/G/X+0M5yvlivIdV4doGXH3aGDYtCD/vGzYUowxafgUG29m +Hpfrp4Gacyv75Rg1NzSRclgsIVswjCEtAQ5Upsi9jtsBrcT4q3B1l3AItQXjVs1S +eUEdShivwiMNw+ypp7wY7JFybd6QofeMuIRTPOwv5P2HIl0HKvF77SQOEllUJ/ai +MCvsvYYVAoGBAOdQjqR+wcA0gMUsH0AdRcRj6MIeMPFNuApGCP5JnAl7kUk7d1iM +HF5l3af41Rv8K58/VMkTRtg7/lfSvnXb3Do/8RjiNCYrSEzY1wSETXWsQLm7o2BR +Pd3VjRydqWDnNnbynmhN1QSX/wm+C0A+/ADZTv2I+ronSckYnYsRc5K/AoGBAMje +qv4FN0MaRTuwvQF64GH3lRu7tyDebi8vXx7qFHoGKuNeqLN1n7rzxGv+CW3WMe64 +XFv7uj8tTlIRmOo0zZRcX/RAzZzksYVzuRzt5JAiII39Om/t8R+a1TIcyxCqNu56 +AgbLFlr1ifV2AdPhL/jaVDaOQuYtRWQbalRwo6FvAoGASWoH261fUOFY/TtDr9Wa +jvompXVvkyZXgus3b31tSJUvR07YUgIl/s/Vybnv6THfpmszPZ4gngBsePMp/74H +53Tj0EH6yyJxj/JCS+yxqOdCo7Ap6ifeuslMQAjJ+Tdo0deu7uvE7/BNwYnPiFR1 +Js/RgiY3bV2KzYhCeCDRzYECgYBHCkkbMakLuFzH8YfG4Dg/LZq0hj0MCN6bHXvc +qRi41zmP1EiBv5XQPqo3L08SY3ChZt5zQaHq+hD7AXI9UpNqZwkgj1KorFEES5sq +WhA90XQGA4sJTvO6I75GzNwcdraavNe+o1CytqgmAag/0SJwNncWHFYYpeYms3N1 +jaixTQKBgQDR/UeoGjnxwXyHyayw+NKjHCd6zMsy95CbRGkmipJGZFZ76Er8Dvw3 ++BqYJLHZsGzPLx1lBNlK/6Q3ysI9NLaAEzLK1+aSquA+yYZgWxBT7S5YWseL3wVJ +2h2B/dxyhzp892ElHM6Kvq0cxc2IEiVGntBQwCxcoToPgt17mXW9SA== +-----END RSA PRIVATE KEY----- diff --git a/test/tls.js b/test/tls.js new file mode 100644 index 0000000..6416a22 --- /dev/null +++ b/test/tls.js @@ -0,0 +1,92 @@ +var config = require('./config.js'); +var clientConfig = JSON.parse(JSON.stringify(config)); // clone object +clientConfig.port = 1000 + Math.floor(Math.random()*1000); +clientConfig.tls = true; + +var tls = require('tls'); +var net = require('net'); +var fs = require('fs'); +var path = require('path'); +var tlsOpts = { + key: fs.readFileSync(path.join(__dirname, 'certs/my-server.key.pem')), + cert: fs.readFileSync(path.join(__dirname, './certs/my-server.crt.pem')), + ca: fs.readFileSync(path.join(__dirname, './certs/my-private-root-ca.crt.pem')) +}; +tls.createServer(tlsOpts, function (socket) { + var conn = net.createConnection(config.port, config.host); + socket.pipe(conn).pipe(socket); +}).listen(clientConfig.port); + +process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; // ignore self-signed certificates + +var r = require('../lib')(clientConfig); +var util = require(__dirname+'/util/common.js'); +var assert = require('assert'); +var uuid = util.uuid; +var It = util.It; + +var dbName = uuid(); +var tableName = uuid(); +It("Init for `tls.js`", function *(done) { + try { + var result = yield r.dbCreate(dbName).run(); + assert.equal(1, result.dbs_created); + var tableCreated = yield r.db(dbName).tableCreate(tableName)('tables_created').run() + assert.equal(1, tableCreated); + done(); + } + catch(e) { + done(e); + } +}); + +It("hello world for `tls.js`", function *(done) { + try { + var doc = { + "id": "hello world" + }; + + var result = yield r.db(dbName).table(tableName).insert(doc).run(); + assert.equal(1, result.inserted); + + var docFromDb = yield r.db(dbName).table(tableName).get('hello world').run(); + assert.deepEqual(doc, docFromDb); + done(); + } + catch(e) { + done(e); + } +}); + +It("hello world for `tls.js` with TLS options", function *(done) { + process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; // do not ignore self-signed certificates (default) + clientConfig.tls = tlsOpts; + r = require('../lib')(clientConfig); + try { + var doc = { + "id": "hello world safe!" + }; + + var result = yield r.db(dbName).table(tableName).insert(doc).run(); + assert.equal(1, result.inserted); + + var docFromDb = yield r.db(dbName).table(tableName).get('hello world safe!').run(); + assert.deepEqual(doc, docFromDb); + done(); + } + catch(e) { + done(e); + } +}); + +It("cleanup for TLS options", function *(done) { + try { + var result = yield r.dbDrop(dbName).run(); + assert.equal(1, result.dbs_dropped); + assert.equal(1, result.tables_dropped); + done(); + } + catch(e) { + done(e); + } +}); \ No newline at end of file From fa15484d83907bb64d270e90019baf5f30657eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sat, 28 Feb 2015 21:08:36 +0000 Subject: [PATCH 002/304] Make all paths are the same --- test/tls.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tls.js b/test/tls.js index 6416a22..26a848d 100644 --- a/test/tls.js +++ b/test/tls.js @@ -9,8 +9,8 @@ var fs = require('fs'); var path = require('path'); var tlsOpts = { key: fs.readFileSync(path.join(__dirname, 'certs/my-server.key.pem')), - cert: fs.readFileSync(path.join(__dirname, './certs/my-server.crt.pem')), - ca: fs.readFileSync(path.join(__dirname, './certs/my-private-root-ca.crt.pem')) + cert: fs.readFileSync(path.join(__dirname, 'certs/my-server.crt.pem')), + ca: fs.readFileSync(path.join(__dirname, 'certs/my-private-root-ca.crt.pem')) }; tls.createServer(tlsOpts, function (socket) { var conn = net.createConnection(config.port, config.host); From 7da276c74d30940cb21e2bb26b4787f71ff6b773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:23:09 +0000 Subject: [PATCH 003/304] Pass TLS option to connection options --- lib/pool.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pool.js b/lib/pool.js index 64b145c..622469d 100644 --- a/lib/pool.js +++ b/lib/pool.js @@ -17,7 +17,6 @@ function Pool(r, options) { this.options.maxExponent = options.maxExponent || 6; // Maximum timeout is 2^maxExponent*timeoutError this.options.silent = options.silent || false; // Maximum timeout is 2^maxExponent*timeoutError - this.options.connection = { host: options.host || this._r._host, port: options.port || this._r._port, @@ -25,7 +24,8 @@ function Pool(r, options) { timeout: options.timeout || this._r._timeoutConnect, authKey: options.authKey || this._r._authKey, cursor: options.cursor || false, - stream: options.stream || false + stream: options.stream || false, + tls: options.tls || false } this._pool = new Dequeue(this.options.buffer+1); From cd1303deb24c4264ae7afadafb8f6949b998a866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:23:33 +0000 Subject: [PATCH 004/304] Test with higher ports --- test/tls.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tls.js b/test/tls.js index 26a848d..8076f9d 100644 --- a/test/tls.js +++ b/test/tls.js @@ -1,6 +1,6 @@ var config = require('./config.js'); var clientConfig = JSON.parse(JSON.stringify(config)); // clone object -clientConfig.port = 1000 + Math.floor(Math.random()*1000); +clientConfig.port = 10000 + Math.floor(Math.random()*1000); clientConfig.tls = true; var tls = require('tls'); From c33f4004b610b45fa4d0d26df0a6d4651e4f794f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:26:46 +0000 Subject: [PATCH 005/304] Delete useless certs --- test/certs/my-private-root-ca.key.pem | 27 --------------------------- test/certs/my-private-root-ca.srl | 1 - test/certs/my-server.csr.pem | 17 ----------------- 3 files changed, 45 deletions(-) delete mode 100644 test/certs/my-private-root-ca.key.pem delete mode 100644 test/certs/my-private-root-ca.srl delete mode 100644 test/certs/my-server.csr.pem diff --git a/test/certs/my-private-root-ca.key.pem b/test/certs/my-private-root-ca.key.pem deleted file mode 100644 index 89a31b6..0000000 --- a/test/certs/my-private-root-ca.key.pem +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAxzw2YVJ766RiHgZHtkfSASzxiiYj4z1/DFgi4jaHcIaL5XMk -PIxPPz/v/aTVNCug9AaBX8zs/GGlIp6EhpAdjyOa9ZXI0gL/J82Q/EbxcUmB8SDF -fKBbda4wPw6FJ4FJLILgykliEDN0H7/dxsnWQTvBj+hXlkT57GiWQbik/wVhm+F1 -RVqu8hZ44UOEVoe6NkOhCIfPQSV+2c8XWPa5wdWNDJiEL/WWLPZDCn3M9sFkbYp8 -S19B7a5zVWjeyiigzHYjAkdH/qGmoGfhjeMEBdksRIk2EZmWdmO7TGTnTec69zxr -O37WKSsJ4Kr7WksYfe5nTvAehb6B0sK86yPSUwIDAQABAoIBACHamWLwIR5pUELJ -zmWqzoRknqZa1L5INM5kK4lEur7kHeFfL0kajlyxAJaY66FxyGeus00UBTDdZrH7 -PYmf82XfJvWT/guxdqnMxFYZt+0IEIxohSYoGWJlts0AKqAL1+M4WfoFfiRCmfjL -IWQl58It5TYOxNND5TPGkcGZHnA2rNa+9SV8ILELCsTYL8dKI07WEGiFqsmrGTfv -O+8QlGtE2i97ze1+8tcQ0XbqM8SAncq2LIq5NHmulds0fBnm4Z3JI0YpaZXS60xu -AA040AJNWnkIg3j+MFyM48tWuhm5QnqZc+rg9tJKo9UaTsZeGVGTUS3k6uOuknIl -/Px5hAECgYEA9hPFC3SYBD5jWZGE6YOT61rAlDV4MypZJfG9lgP2qSvL2ceSjy3i -4dgU80rpz5DWYrkifGOL8D/JIiGA0SmKp716hmA4Iq5NlyA9jAwGzJ7Fv2133a6V -NujXzw7gd0tyjIP1u+EhSTpAOSqt/a4rCn/J0enI+LgWJ+JjOMRQlCcCgYEAz0Tl -tAMfNueo63O4+zQyF9i4xMPXQf1Imk7c0Y4XhmH2cs1YaEned4DvF6mb1hJSrZUM -+3ic5pQUCltkky6GiSKZA0UT7uUBpNlUtrh2BvaVtMkFp8zo8JfomsOHQUld9tJE -JPRePz/NnfVLnUn9fLZ/AMUDRICRCb0Ng1wCT/UCgYA9N83de7QV2i99KOaX8VPr -STh/po2wrOhguDMiDvpeO7cLBss/M3suFEOiAxEHwlCTXttldp5ptnSjORKC9oK4 -GDtTErzkY9iJsEufe551aUpoxSL802HrtyF5MYpGI07sBigsBWXygbtYfXWrlpi+ -k+vxfOeyeJmKcpecl7o0IQKBgG7ab2wOe2RXFA4Rj26y5NPCMlfyUi9V8r2szgc5 -0rUpsbCGyPDGCTiq+bUTHmio1hVGcXIReQENU4aSiMD7EjNIEMQh7t5HFtD+94Le -NynUOg89sEulmTQMWvhi+PTe82Vt4cpN9BYp3qPLrIxMJUnNJTHZLL03cdrehNqr -nAT9AoGBAL1pa0z6Edhy539mKGBDdlWpDseakyD4hMwlSf633s8bbOtGNiU4lON3 -yc3eCqtsDFy1Ml/QyYsh7zE9omR8+2H841ubEXOaocEXYlg0lqQOgsowXg73C/sy -cDpEl84WgUufXJLFL8m7TnjFUTpIU+bIU60UtzQYmhJDQfGBMTfD ------END RSA PRIVATE KEY----- diff --git a/test/certs/my-private-root-ca.srl b/test/certs/my-private-root-ca.srl deleted file mode 100644 index 7d09bb5..0000000 --- a/test/certs/my-private-root-ca.srl +++ /dev/null @@ -1 +0,0 @@ -88E48FE6D1A917DB diff --git a/test/certs/my-server.csr.pem b/test/certs/my-server.csr.pem deleted file mode 100644 index 61f5b60..0000000 --- a/test/certs/my-server.csr.pem +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE REQUEST----- -MIICnTCCAYUCAQAwWDELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFV0YWgxDjAMBgNV -BAcMBVByb3ZvMRYwFAYDVQQKDA1BQ01FIFRlY2ggSW5jMRIwEAYDVQQDDAlsb2Nh -bGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1gCHNOwqPSc3k -ewem2fR4njKL1QqLSH2qfNGjG0RU/LWEB+u4YJCPPoUDb0I1Q/fA8S6IQIIf5xaJ -VHpzlZ2tFLR0dfvcxPujD9Nog0hLxBcKRNeAxDFzonv7cNmPxCWK6rFEOOE5z31e -L0+AgFdkdYw8fHKHWKqNJPlfRZBGkvPuLEYf1209EItQqkOU8DqUBSgKQXnKKb8z -yh4MXJKYVUUpKKhgkMvv6FxsDNLLnXcw/2StN8G1UPMtC5+yPCjFmvg6jCi+6Jlk -+a5KyiV4m3K9XHfv8wYa9gZTYJnXdwyp9yR+fx6XvtYWao8QdrNbzICSTbmj6X7D -rrxAA7/RAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAGOt0xMNt7HB9QWMw0A9Q -bJgazLmJunvO5hpV+77cmyKs29w1DVGhEzE6fjporngkVz1Flso0NFYVK2oLhgfu -Q8QHEgm1a1T7hHuiPFk6sLNS/Ahq565O3VXzRp34OKhino7N9AUc2XXeFf8UF+jb -KRJtqCqRvqO/Ukzc579W6fOqGpP/ojoG8tOsDFg2JKrLT5rp3CM9vYVFpirAg/EV -dRKIZHReV6Hh0VM3Be1QCFtVeFvPvtm5BJ6DYvJpr7k4MqdCR6nZ3u652BinU1BB -E/NTSL1As0gHTk9r475ExXFQ+meABunVmO83ppo7sp42mZHFXzqFbUAN9hdp59o3 -HA== ------END CERTIFICATE REQUEST----- From 4d94c628c8899ba2f06848ecc5f69a4af1366887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:42:11 +0000 Subject: [PATCH 006/304] Document tls option --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 0d8bad8..8c77b90 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,38 @@ removing `toArray`: ``` +#### Using TLS Connections + +RethinkDB does not support TLS connections to the server yet, but in case you want +to run it over an untrusted network and need encryption, you can easily run a TLS proxy +on your server with: + +```js +var tls = require('tls'); +var net = require('net'); +var tlsOpts = { + key: fs.readFileSync('private-key.pem'), + cert: fs.readFileSync('public-cert.pem') +}; +tls.createServer(tlsOpts, function (encryptedConnection) { + encryptedConnection.pipe(net.connect(28015)).pipe(encryptedConnection); +}).listen(29015); +``` + +And then connect to it safely with the `tls` option: + +```js +var r = require('rethinkdbdash')({ + port: 29015, + host: 'place-with-no-firewall.com', + tls: true +}); +``` + +`tls` may also be an object that will be passed as the `options` argument to +[`tls.connect`](http://nodejs.org/api/tls.html#tls_tls_connect_options_callback) +so you can for instance pass a `ca` if you are using a self signed certificate. + ### New features and differences From 0882d9d38a00778e4192d44dae71493c97422200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:43:20 +0000 Subject: [PATCH 007/304] Require fs too --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8c77b90..c2af8b9 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,7 @@ on your server with: ```js var tls = require('tls'); var net = require('net'); +var fs = require('fs'); var tlsOpts = { key: fs.readFileSync('private-key.pem'), cert: fs.readFileSync('public-cert.pem') From 8eefb434b1b5893eb1b214b9440cd8a831fea398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jer=C3=B3nimo?= Date: Sun, 1 Mar 2015 00:51:47 +0000 Subject: [PATCH 008/304] Do not test the self signed certificate on werker --- test/tls.js | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/test/tls.js b/test/tls.js index 8076f9d..dfb42e4 100644 --- a/test/tls.js +++ b/test/tls.js @@ -58,26 +58,33 @@ It("hello world for `tls.js`", function *(done) { } }); -It("hello world for `tls.js` with TLS options", function *(done) { +/* + The self signed certificate is only good for localhost and not + other IP/hostnames that may be used in CIs, so we will only test + this if the environment variable WERCKER_RETHINKDB_HOST is not declared. +*/ +if (clientConfig.host === 'localhost') { process.env.NODE_TLS_REJECT_UNAUTHORIZED = "1"; // do not ignore self-signed certificates (default) clientConfig.tls = tlsOpts; r = require('../lib')(clientConfig); - try { - var doc = { - "id": "hello world safe!" - }; + It("hello world for `tls.js` with TLS options", function *(done) { + try { + var doc = { + "id": "hello world safe!" + }; - var result = yield r.db(dbName).table(tableName).insert(doc).run(); - assert.equal(1, result.inserted); + var result = yield r.db(dbName).table(tableName).insert(doc).run(); + assert.equal(1, result.inserted); - var docFromDb = yield r.db(dbName).table(tableName).get('hello world safe!').run(); - assert.deepEqual(doc, docFromDb); - done(); - } - catch(e) { - done(e); - } -}); + var docFromDb = yield r.db(dbName).table(tableName).get('hello world safe!').run(); + assert.deepEqual(doc, docFromDb); + done(); + } + catch(e) { + done(e); + } + }); +} It("cleanup for TLS options", function *(done) { try { From 2c7de90b4a0d5421083e3db19e7c72f8190b6bb6 Mon Sep 17 00:00:00 2001 From: Michel Date: Sun, 8 Mar 2015 00:44:03 -0800 Subject: [PATCH 009/304] Add support for connection pools with multiple hosts --- README.md | 62 +- lib/connection.js | 12 + lib/dequeue.js | 1 + lib/helper.js | 37 + lib/index.js | 27 +- lib/pool.js | 39 +- lib/pool_master.js | 328 ++++++ lib/term.js | 6 +- test/config.js | 7 +- test/multiple-require.js | 2 +- test/pool.js | 755 +++++++++----- test/pool_legacy.js | 346 +++++++ test/stream.js | 33 +- test/util/common.js | 7 +- test/util/fake_server/README.md | 5 + test/util/fake_server/database.js | 21 + test/util/fake_server/document.js | 133 +++ test/util/fake_server/error.js | 39 + test/util/fake_server/group.js | 74 ++ test/util/fake_server/helper.js | 735 +++++++++++++ test/util/fake_server/index.js | 245 +++++ test/util/fake_server/node.js | 18 + test/util/fake_server/protodef.js | 206 ++++ test/util/fake_server/query.js | 1556 ++++++++++++++++++++++++++++ test/util/fake_server/selection.js | 197 ++++ test/util/fake_server/sequence.js | 829 +++++++++++++++ test/util/fake_server/table.js | 519 ++++++++++ 27 files changed, 5939 insertions(+), 300 deletions(-) create mode 100644 lib/pool_master.js create mode 100644 test/pool_legacy.js create mode 100644 test/util/fake_server/README.md create mode 100644 test/util/fake_server/database.js create mode 100644 test/util/fake_server/document.js create mode 100644 test/util/fake_server/error.js create mode 100644 test/util/fake_server/group.js create mode 100644 test/util/fake_server/helper.js create mode 100644 test/util/fake_server/index.js create mode 100644 test/util/fake_server/node.js create mode 100644 test/util/fake_server/protodef.js create mode 100644 test/util/fake_server/query.js create mode 100644 test/util/fake_server/selection.js create mode 100644 test/util/fake_server/sequence.js create mode 100644 test/util/fake_server/table.js diff --git a/README.md b/README.md index 0d8bad8..265e79b 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ here are the few things to do: ```js var r = require('rethinkdbdash')(); // Or if you do not connect to the default local instance: - // var r = require('rethinkdbdash')({host: ..., port: ...}); + // var r = require('rethinkdbdash')(servers: [{host: ..., port: ...}]); ``` 2. Remove everything related to a connection: @@ -145,19 +145,53 @@ When you import the driver, as soon as you execute the module, you will create a default connection pool (except if you pass `{pool: false}`. The options you can pass are: -- `{pool: false}` -- if you do not want to use a connection pool. -- the options for the connection pool, which can be: +- `auto`: `` - When true, the driver will regularly pull data from the table `server_status` to +keep a list of updated hosts, default `true` +- `pool`: `` - Set it to `false`, if you do not want to use a connection pool. +- `buffer`: `` - Minimum number of connections available in the pool, default `50` +- `max`: ``` - Maximum number of connections available in the pool, default `1000` +- `timeout`: ` - The number of seconds for a connection to be opened, default `20` +- `timeoutError`: ` - Wait time before reconnecting in case of an error (in ms), default 1000 +- `imeoutGb`: `` - How long the pool keep a connection that hasn't been used (in ms), default 60*60*1000 +- `maxExponent`: `` - The maximum timeout before trying to reconnect is 2^maxExponent x timeoutError, default 6 (~60 seconds for the longest wait) +- `silent`: - console.error errors, default `false` +- `servers`: an of objects `{host: , port: }` representing instances of +RethinkDB to initially connect to. + +In case of a single instance, you can directly pass `host` and `port` in the top level parameters. + +Examples: +``` +// connect to localhost:8080, and let the driver find other instances +var r = require('rethinkdbdash')(); -```js -{ - buffer: , // minimum number of connections available in the pool, default 50 - max: , // maximum number of connections in the pool, default 1000 - timeout: , // number of seconds for a connection to be opened, default 20 - timeoutError: , // wait time before reconnecting in case of an error (in ms), default 1000 - timeoutGb: , // how long the pool keep a connection that hasn't been used (in ms), default 60*60*1000 - maxExponent: , // the maximum timeout before trying to reconnect is 2^maxExponent*timeoutError, default 6 (~60 seconds for the longest wait) - silent: // console.error errors (default false) -} +// connect to and only to localhost:8080 +var r = require('rethinkdbdash')({ + auto: false +}); + +// Do not create a connection pool +var r = require('rethinkdbdash')({pool: false}); + +// Connect to a cluster seeding from `192.168.0.100`, `192.168.0.100`, `192.168.0.102` +var r = require('rethinkdbdash')({ + servers: [ + {host: '192.168.0.100', port: 28015}, + {host: '192.168.0.101', port: 28015}, + {host: '192.168.0.102', port: 28015}, + ] +}); + +// Connect to a cluster containing `192.168.0.100`, `192.168.0.100`, `192.168.0.102` +var r = require('rethinkdbdash')({ + servers: [ + {host: '192.168.0.100', port: 28015}, + {host: '192.168.0.101', port: 28015}, + {host: '192.168.0.102', port: 28015}, + ], + buffer: 300, + max: 3000 +}); ``` You can also pass `{cursor: true}` if you want to retrieve RethinkDB streams as cursors @@ -171,7 +205,7 @@ As mentionned before, `rethinkdbdash` has a connection pool and manage all the c itself. The connection pool is initialized as soon as you execute the module. You should never have to worry about connections in rethinkdbdash. Connections are created -as they are needed, and in case of failure, the pool will try to open connections with an +as they are needed, and in case of a host failure, the pool will try to open connections with an exponential back off algorithm. The driver will execute one query per connection as queries are not executed in parallel diff --git a/lib/connection.js b/lib/connection.js index 3c90232..aa606db 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -51,12 +51,16 @@ function Connection(r, options, resolve, reject) { // We emit end or close just once self.connection.removeAllListeners(); self.emit('end'); + // We got a FIN packet, so we'll just flush + self._flush(); }); self.connection.on("close", function(error) { // We emit end or close just once clearTimeout(self.timeoutOpen) self.connection.removeAllListeners(); self.emit('closed'); + // The connection is fully closed, flush (in case 'end' was not triggered) + self._flush(); }); self.connection.setNoDelay(); self.connection.on("connect", function() { @@ -578,5 +582,13 @@ Connection.prototype._isOpen = function() { return this.open; } +Connection.prototype._flush = function() { + helper.loopKeys(this.rejectMap, function(rejectMap, key) { + rejectMap[key](new Error("The connection was closed before the query could be completed.")) + }); + this.rejectMap = {}; + this.resolveMap = {}; +} + module.exports = Connection diff --git a/lib/dequeue.js b/lib/dequeue.js index 956bf0d..265a303 100644 --- a/lib/dequeue.js +++ b/lib/dequeue.js @@ -73,6 +73,7 @@ Dequeue.prototype.push = function(element) { } Dequeue.prototype.pop = function(element) { + //TODO: Decrease size when possible/needed // Return the element in this.end-1 if (this.getLength() > 0) { var pos = this.end-1; diff --git a/lib/helper.js b/lib/helper.js index bc0fe5a..b8b3d7b 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -99,3 +99,40 @@ function changeProto(object, other) { object.__proto__ = other.__proto__; } module.exports.changeProto = changeProto; + +// Try to extract the most global address +// Note: Mutate the input +function getCanonicalAddress(addresses) { + // We suppose that the addresses are all valid, and therefore use loose regex + for(var i=0; i max) { + result = addresses[i]; + max = addresses[i].value; + } + } + return result; +} +module.exports.getCanonicalAddress = getCanonicalAddress; diff --git a/lib/index.js b/lib/index.js index 870a251..05333b1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -4,7 +4,7 @@ var helper = require(__dirname+"/helper.js"); var Connection = require(__dirname+"/connection.js"); var Term = require(__dirname+"/term.js"); var Error = require(__dirname+"/error.js"); -var Pool = require(__dirname+"/pool.js"); +var PoolMaster = require(__dirname+"/pool_master.js"); var termTypes = require(__dirname+"/protodef.js").Term.TermType; function r(options) { @@ -79,13 +79,26 @@ r.prototype.connect = function(options, callback) { return p; }; -r.prototype.createPool = function(options) { - this._pool = new Pool(this, options); - +r.prototype.createPools = function(options) { + this._poolMaster = new PoolMaster(this, options); return this; } -r.prototype.getPool = function() { - return this._pool; + +r.prototype.getPoolMaster = function() { + return this._poolMaster; +} +r.prototype.getPool = function(i) { + if (i === undefined) { + if (this.getPoolMaster().getPools().length === 1) { + return this.getPoolMaster().getPools()[0]; + } + else { + throw new Error("You have multiple pools. Use `getPool(index)` or `getPools()`"); + } + } + else { + return this.getPoolMaster().getPools()[i]; + } } r.prototype.expr = function(expression, nestingLevel) { @@ -472,7 +485,7 @@ function main(options) { var _r = new r(); if (!helper.isPlainObject(options)) options = {}; - if (options.pool !== false) _r.createPool(options); + if (options.pool !== false) _r.createPools(options); _r._options = {}; if (options.cursor === true) _r._options.cursor = true; if (options.stream === true) _r._options.stream = true; diff --git a/lib/pool.js b/lib/pool.js index 64b145c..5ca84fd 100644 --- a/lib/pool.js +++ b/lib/pool.js @@ -29,7 +29,8 @@ function Pool(r, options) { } this._pool = new Dequeue(this.options.buffer+1); - this._line = new Dequeue(this.options.buffer+1); + //TODO: This logic should be moved in the master + //this._line = new Dequeue(this.options.buffer+1); this._draining = null; this._numConnections = 0; @@ -72,12 +73,15 @@ Pool.prototype.getConnection = function() { return reject(new Err.ReqlDriverError("The pool does not have any opened connections and failed to open a new one")); } + /* self._line.push({ resolve: resolve, reject: reject }); + */ - self.emit('queueing', self._line.getLength()) + + //self.emit('queueing', self._line.getLength()) } if (self._slowGrowth === false) { @@ -94,10 +98,17 @@ Pool.prototype._decreaseNumConnections = function() { if ((this._draining !== null) && (this._numConnections === 0)) { this._draining.resolve(); } + if (this._numConnections === 0) { + this.emit('empty'); + } } Pool.prototype._increaseNumConnections = function() { this._numConnections++; this.emit('size', this._numConnections) + if (this._numConnections === 1) { + this.emit('not-empty'); + } + } @@ -111,6 +122,7 @@ Pool.prototype.putConnection = function(connection) { } } else { + /* if (self._line.getLength() > 0) { clearTimeout(connection.timeout); @@ -120,8 +132,10 @@ Pool.prototype.putConnection = function(connection) { self.emit('queueing', self._line.getLength()) } else { + */ self._pool.push(connection); self.emit('available-size', self._pool.getLength()); + self.emit('new-connection', connection); var timeoutCb = function() { if (self._pool.get(0) === connection) { @@ -139,7 +153,7 @@ Pool.prototype.putConnection = function(connection) { } } connection.timeout = setTimeout(timeoutCb, self.options.timeoutGb); - } + //} } }; @@ -233,11 +247,13 @@ Pool.prototype.createConnection = function() { self._openingConnections--; self._decreaseNumConnections(); + /* if (self._numConnections === 0) { while(self._line.getLength() > 0) { self._line.shift().reject(new Err.ReqlDriverError("The pool does not have any opened connections and failed to open a new one")); } } + */ self._slowGrowth = true; if (self._slowlyGrowing === false) { @@ -246,9 +262,11 @@ Pool.prototype.createConnection = function() { self._slowlyGrowing = true; // Log an error - if (self.options.silent !== true) console.error("Fail to create a new connection for the connection pool The error returned was:") - if (self.options.silent !== true) console.error(error.message); - if (self.options.silent !== true) console.error(error.stack); + if (self.options.silent !== true) { + console.error("Fail to create a new connection for the connection pool The error returned was:") + console.error(error.message); + console.error(error.stack); + } if (self._openingConnections === 0) { self._consecutiveFails++; @@ -261,12 +279,15 @@ Pool.prototype.createConnection = function() { }; Pool.prototype._aggressivelyExpandBuffer = function() { - for(var i=0; i 0)) { + setTimeout(function() { self.fetchServers() }, 0); + setInterval(function() { self.fetchServers() }, self._refresh); + } + +} + +PoolMaster.prototype.getPools = function() { + var result = []; + helper.loopKeys(this._pools, function(pools, key) { + if (key === UNKNOWN_POOLS) { + for(var i=0;i 0) { + this._line.shift().reject(new Err.ReqlDriverError("None of the pools have an opened connection and failed to open a new one")); + } +} + +PoolMaster.prototype.getConnection = function() { + var self = this; + var index = self._index; + // Find a pool with available connections + for(var i=0; i 0) { + self._index++; + if (self._index === self._healthyPools.length) { + self._index = 0; + } + return self._healthyPools[index].getConnection(); + } + index++; + if (index === self._healthyPools.length) { + index = 0; + } + } + if (self._healthyPools.length === 0) { + return new Promise(function(resolve, reject) { + reject(new Error("None of the pools have an opened connection and failed to open a new one")); + }); + } + else { + // All pool are busy, buffer the request + return new Promise(function(resolve, reject) { + //console.log('------------'); + //console.log(new Error().stack); + self._line.push({ + resolve: resolve, + reject: reject + }); + // We could add a condition to be less greedy (for early start) + self._expandAll(); + }); + + } +} +PoolMaster.prototype._expandAll = function() { + for(var i=0; i 0) { + var p = self._line.shift(); + this.getConnection().then(p.resolve).error(p.reject); + } + }); + pool.on('not-empty', function() { + var found = false; + for(var i=0; i= expected); + assert(end-start < expected+100); + assert.equal(result, expected); + yield connection.close(); done(); } catch(e) { @@ -30,319 +43,563 @@ It("`createPool` should create a pool and `getPool` should return it", function* } }); -//TODO try to make this tests a little more deterministic -It("`run` should work without a connection if a pool exists", function* (done) { - try { - result = yield r.expr(1).run() - assert.equal(result, 1); +It("Test pool no query with auto: false", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 10, + buffer: 5, + auto: false + }); - assert(r.getPool().getAvailableLength() >= 2); // This can be 2 because r.expr(1) may be run BEFORE a connection in the buffer is available - assert(r.getPool().getAvailableLength() <= r.getPool().getLength()) - assert(r.getPool().getAvailableLength() >=2); - done() + try { + var result = yield util.sleep(200); + // 5 connections are immediately created + assert.equal(r.getPool(0).getLength(), 5); + yield r.getPoolMaster().drain(); + done(); } catch(e) { done(e); } }); -It("The pool should keep a buffer", function* (done) { +It("Test pool no query with auto: true", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 10, + buffer: 5 + }); + try { - result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] - assert.deepEqual(result, [1,1,1,1,1]); - assert(r.getPool().getLength() >= options.buffer+result.length); - - setTimeout( function() { - assert(r.getPool().getAvailableLength() >= result.length) // The connections created for the buffer may not be available yet - assert.equal(r.getPool().getLength(), r.getPool().getLength()) - done(); - }, 500) + var result = yield util.sleep(200); + // 5 connections are immediately created + // 1 connection for fetchServer (via expandAll) + // 1 connection for fetchServer (via getConnection to refill the buffer) + assert.equal(r.getPool(0).getLength(), 7); + yield r.getPoolMaster().drain(); + done(); } catch(e) { done(e); } }); -It("A noreply query should release the connection", function* (done) { +It("Test expanding the pool with auto: false", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 9, + buffer: 5, + auto: false + }); try { - var numConnections = r.getPool().getLength(); - yield r.expr(1).run({noreply: true}) - assert.equal(r.getPool().getLength(), numConnections); + yield util.sleep(1000); + assert.equal(r.getPool().getLength(), 5); + var result = yield [ + r.expr(200).run(), + r.expr(200).run() + ] + assert.equal(result.length, 2); + assert.equal(r.getPool().getLength(), 7); + yield r.getPoolMaster().drain(); done(); } catch(e) { - console.log(e) done(e); } }); -It("The pool shouldn't have more than `options.max` connections", function* (done) { +It("Test expanding the pool with auto: true", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 11, + buffer: 5 + }); try { - result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] - assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) - assert.equal(r.getPool().getLength(), options.max) - - setTimeout( function() { - assert.equal(r.getPool().getAvailableLength(), options.max) - assert.equal(r.getPool().getAvailableLength(), r.getPool().getLength()) - done() - }, 500) + yield util.sleep(300); + assert.equal(r.getPool().getLength(), 7); + var result = yield [ + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run() + ] + assert.equal(result.length, 3); + assert.equal(r.getPool().getLength(), 8); + yield r.getPoolMaster().drain(); + done(); } catch(e) { done(e); } }); - -It("Init for `pool.js`", function* (done) { +It("Test expanding the pool to max with auto: false", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 9, + buffer: 5, + auto: false + }); try { - dbName = uuid(); - tableName = uuid(); - - result = yield r.dbCreate(dbName).run(); - assert.equal(result.dbs_created, 1); - - result = yield r.db(dbName).tableCreate(tableName).run(); - assert.equal(result.tables_created, 1); - - result = yield r.db(dbName).table(tableName).insert(eval('['+new Array(10000).join('{}, ')+'{}]')).run(); - assert.equal(result.inserted, 10000); - pks = result.generated_keys; - + assert.equal(r.getPool().getLength(), 5); + var result = yield [ + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run() + ] + assert.equal(result.length, 10); + assert.equal(r.getPool(0).getLength(), 9); + yield r.getPoolMaster().drain(); done(); } catch(e) { done(e); } -}) -It("Updating data to make it heavier", function* (done) { +}); +It("Test expanding the pool to max with auto: true", function* (done) { + server1.mockServersStatus([server1]) + var r = require(__dirname+'/../lib')({ + host: server1.host, + port: server1.port, + max: 9, + buffer: 5 + }); try { - //Making bigger documents to retrieve multiple batches - var result = yield r.db(dbName).table(tableName).update({ - "foo": uuid(), - "fooo": uuid(), - "foooo": uuid(), - "fooooo": uuid(), - "foooooo": uuid(), - "fooooooo": uuid(), - "foooooooo": uuid(), - "fooooooooo": uuid(), - "foooooooooo": uuid(), - date: r.now() - }).run(); + assert.equal(r.getPool(0).getLength(), 5); + yield util.sleep(100); + assert.equal(r.getPool(0).getLength(), 7); + var result = yield [ + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run(), + r.expr(200).run() + ] + assert.equal(result.length, 11); + assert.equal(r.getPool(0).getLength(), 9); + yield r.getPoolMaster().drain(); done(); } catch(e) { done(e); } -}) - - - -It("The pool should release a connection only when the cursor has fetch everything or get closed", function* (done) { +}); +It("Test multiple pools with late start", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 15*3, + buffer: 5*3 + }); try { - result = yield [r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true})]; - assert.equal(result.length, 10); - assert.equal(r.getPool().getAvailableLength(), 0); - yield result[0].toArray(); - assert.equal(r.getPool().getAvailableLength(), 1); - yield result[1].toArray(); - assert.equal(r.getPool().getAvailableLength(), 2); - yield result[2].close(); - assert.equal(r.getPool().getAvailableLength(), 3); - yield [result[3].close(), result[4].close(), result[5].close(), result[6].close(), result[7].close(), result[8].close(), result[9].close()] + yield util.sleep(500); + // all +1 for expandAll, and the first pool execute fetchServer + // and recrete a connection as the first 5 have not yet returned + var result = {6: 0, 7: 0}; + var pools = r.getPoolMaster().getPools(); + result[pools[0].getLength()]++; + result[pools[1].getLength()]++; + result[pools[2].getLength()]++; + + assert.deepEqual(result, {6: 2, 7: 1}); + var result = yield [ + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run(), + r.expr(400).run() + ] + assert.equal(result.length, 9); + // 8 = 9/3+5 + // 5 = buffer + assert.equal(r.getPool(0).getLength(), 8); + assert.equal(r.getPool(1).getLength(), 8); + assert.equal(r.getPool(2).getLength(), 8); + yield r.getPoolMaster().drain(); done(); } catch(e) { done(e); } }); - -It("The pool should shrink if a connection is not used for some time", function* (done) { - try{ - r.getPool().setOptions({timeoutGb: 100}); - - result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] - - assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) - - setTimeout(function() { - assert.equal(r.getPool().getAvailableLength(), options.buffer) - assert.equal(r.getPool().getLength(), options.buffer) - done() - },400) +It("Test multiple pools with early start", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 15*3, + buffer: 5*3 + }); + try { + assert.equal(r.getPool(0).getLength(), 5); + assert.equal(r.getPool(1).getLength(), 5); + assert.equal(r.getPool(2).getLength(), 5); + // All these queries are fired on an empty pool master, + // so they will each trigger expandAll + // There's also a fetchServer happening + var result = yield [ + r.expr(1000).run(), + r.expr(1000).run() + ] + + // fetchServer: all -> +1 + // queries: all +> + 1 + // server1 returns connection first and execute all the queries + // +1 for each of them as getConnection will call expandBuffer with no available connection + yield util.sleep(100); + + assert.equal(result.length, 2); + var result = {8: 0, 11: 0}; + result[r.getPool(0).getLength()]++; + result[r.getPool(1).getLength()]++; + result[r.getPool(2).getLength()]++; + + assert.deepEqual(result, {8: 2, 11: 1}); + + yield r.getPoolMaster().drain(); + done(); } catch(e) { done(e); } }); +It("Test multiple pools - kill a server - check options", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 10*3, + buffer: 4*3, + silent: true + }); + try { + yield util.sleep(100); + server2.close(); + yield util.sleep(1000); + assert.equal(r.getPool(0).options.max, 15); + assert.equal(r.getPool(1).options.max, 10); + assert.equal(r.getPool(2).options.max, 15); + assert.equal(r.getPool(0).options.buffer, 6); + assert.equal(r.getPool(1).options.buffer, 4); + assert.equal(r.getPool(2).options.buffer, 6); + + yield r.getPoolMaster().drain(); + // Restart server2 since we killed it + server2 = new Server({ + host: 'localhost', + port: server2.port + }) -It("`pool.drain` should eventually remove all the connections", function* (done) { - try{ - yield r.getPool().drain(); - - assert.equal(r.getPool().getAvailableLength(), 0) - assert.equal(r.getPool().getLength(), 0) - - done() + done(); } catch(e) { done(e); } }); - -It("If the pool cannot create a connection, it should reject queries", function* (done) { +It("Test multiple pools - kill a server while running queries", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 10*3, + buffer: 4*3, + silent: true + }); try { - var r = require(__dirname+'/../lib')({host: "notarealhost", buffer: 1, max: 2, silent: true}); - yield r.expr(1).run() - done(new Error("Was expecting an error")); + yield util.sleep(100); + var success = 0; + var error = 0; + for(var i=0; i<9; i++) { + r.expr(100).run().then(function() { + success++; + }).error(function() { + error++; + }); + } + server2.destroy(); + yield util.sleep(1000); + assert.equal(r.getPool(0).options.max, 15); + assert.equal(r.getPool(1).options.max, 10); + assert.equal(r.getPool(2).options.max, 15); + assert.equal(r.getPool(0).options.buffer, 6); + assert.equal(r.getPool(1).options.buffer, 4); + assert.equal(r.getPool(2).options.buffer, 6); + + assert.equal(success, 6); + assert.equal(error, 3); + + yield r.getPoolMaster().drain(); + // Restart server2 since we killed it + server2 = new Server({ + host: 'localhost', + port: server2.port + }) + + done(); } catch(e) { - if (e.message === "The pool does not have any opened connections and failed to open a new one.") { - done() - } - else { - done(e); - } + done(e); } }); - -It("If the pool cannot create a connection, it should reject queries - timeout", function* (done) { +It("Test multiple pools - kill a server and restart it - auto: true", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 10*3, + buffer: 4*3, + silent: true + }); try { - var r = require(__dirname+'/../lib')({host: "notarealhost", buffer: 1, max: 2, silent: true}); - yield new Promise(function(resolve, reject) { setTimeout(resolve, 1000) }); - yield r.expr(1).run() - done(new Error("Was expecting an error")); + yield util.sleep(100); + var success = 0; + var error = 0; + for(var i=0; i<9; i++) { + r.expr(100).run().then(function() { + success++; + }).error(function() { + error++; + }); + } + server2.destroy(); + yield util.sleep(1000); + server2 = new Server({ + host: server2.host, + port: server2.port + }) + + yield util.sleep(2000); + assert.equal(r.getPool(0).options.max, 10); + assert.equal(r.getPool(1).options.max, 10); + assert.equal(r.getPool(2).options.max, 10); + assert.equal(r.getPool(0).options.buffer, 4); + assert.equal(r.getPool(1).options.buffer, 4); + assert.equal(r.getPool(2).options.buffer, 4); + + assert.equal(success, 6); + assert.equal(error, 3); + + yield r.getPoolMaster().drain(); + // Restart server2 since we killed it + done(); } catch(e) { - if (e.message === "The pool does not have any opened connections and failed to open a new one.") { - done() - } - else { - done(e); - } + done(e); } }); - -It("If the pool is drained, it should reject queries", function* (done) { +It("Test multiple pools - kill a server and restart it - auto: false", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port}, + {host: server2.host, port: server2.port}, + {host: server3.host, port: server3.port} + ], + max: 10*3, + buffer: 4*3, + silent: true, + auto: false + }); try { - var r = require(__dirname+'/../lib')({buffer: 1, max: 2, silent: true}); - - r.getPool().drain(); + yield util.sleep(100); + var success = 0; + var error = 0; + for(var i=0; i<9; i++) { + r.expr(300).run().then(function(result) { + success++; + }).error(function() { + error++; + }); + } + server2.destroy(); + yield util.sleep(1000); + server2 = new Server({ + host: server2.host, + port: server2.port + }) + + yield util.sleep(2000); + assert.equal(r.getPool(0).options.max, 10); + assert.equal(r.getPool(1).options.max, 10); + assert.equal(r.getPool(2).options.max, 10); + assert.equal(r.getPool(0).options.buffer, 4); + assert.equal(r.getPool(1).options.buffer, 4); + assert.equal(r.getPool(2).options.buffer, 4); + + assert.equal(success, 6); + assert.equal(error, 3); + + yield r.getPoolMaster().drain(); + // Restart server2 since we killed it + server2 = new Server({ + host: 'localhost', + port: server2.port + }) - var result = yield r.expr(1).run(); - done(new Error("Was expecting an error")); + done(); } catch(e) { - if (e.message === "The pool is being drained.") { - done() - } - else { - done(e); - } + done(e); } }); - -It("`drain` should work in case of failures", function* (done) { +It("Test adding a new server", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + server1.mockServersStatus([server1, server2]) + server2.mockServersStatus([server1, server2]) + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port} + ], + max: 10*3, + buffer: 4*3, + silent: true, + auto: true + }); try { - r = r.createPool({ - port: 80, // non valid port - silent: true, - timeoutError: 100 - }); - var pool = r.getPool(); - // Sleep 1 sec - yield new Promise(function(resolve, reject) { setTimeout(resolve, 150) }); - pool.drain(); - - // timeoutReconnect should have been canceled - assert.equal(pool.timeoutReconnect, null); - pool.options.silent = false; - yield new Promise(function(resolve, reject) { setTimeout(resolve, 1000) }); + assert.equal(r.getPoolMaster().getPools().length, 1); + yield util.sleep(1000); + assert.equal(r.getPoolMaster().getPools().length, 2); + + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + + r.getPoolMaster().fetchServers(); + yield util.sleep(1000); + assert.equal(r.getPoolMaster().getPools().length, 3); done(); } catch(e) { done(e); } }); - - -/* -It("The pool should remove a connection if it errored", function* (done) { - try{ - r.getPool().setOptions({timeoutGb: 60*60*1000}); - - result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] - - assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) - - // This query will make the error return an error -1 - result = yield r.expr(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) - .run() - - +It("Test removing a new server", function* (done) { + server1.cleanMockServersStatus(); + server2.cleanMockServersStatus(); + server3.cleanMockServersStatus(); + server1.mockServersStatus([server1, server2, server3]) + server2.mockServersStatus([server1, server2, server3]) + server3.mockServersStatus([server1, server2, server3]) + var r = require(__dirname+'/../lib')({ + hosts: [ + {host: server1.host, port: server1.port} + ], + max: 10*3, + buffer: 4*3, + silent: true, + auto: true + }); + try { + assert.equal(r.getPoolMaster().getPools().length, 1); + yield util.sleep(1000); + assert.equal(r.getPoolMaster().getPools().length, 3); + + yield util.sleep(1000); + server1.mockServersStatus([server1, server3]) + server3.mockServersStatus([server1, server3]) + server2.close(); + + //TODO Why doesn't server2 doesn't trigger empty and fetchServers? + r.getPoolMaster().fetchServers(); + yield util.sleep(1000); + assert.equal(r.getPoolMaster().getPools().length, 2); + done(); } catch(e) { - if ((true) || (e.message === "Client is buggy (failed to deserialize protobuf)")) { - // We expect the connection that errored to get closed in the next second - setTimeout(function() { - assert.equal(r.getPool().getAvailableLength(), options.max-1) - assert.equal(r.getPool().getLength(), options.max-1) - done() - }, 1000) - } - else { - done(e); - } - + done(e); } }); -*/ - - diff --git a/test/pool_legacy.js b/test/pool_legacy.js new file mode 100644 index 0000000..2d578ea --- /dev/null +++ b/test/pool_legacy.js @@ -0,0 +1,346 @@ +var config = require(__dirname+'/config.js'); +var r = require(__dirname+'/../lib')({pool: false, silent: true}); +var util = require(__dirname+'/util/common.js'); +var assert = require('assert'); +var Promise = require('bluebird'); + +var uuid = util.uuid; +var It = util.It; + +var uuid = util.uuid; +var dbName, tableName, result, pks; + +var options = { + max: 10, + buffer: 2, + host: config.host, + port: config.port, + authKey: config.authKey, + auto: false + //silent: true +}; + +It("`createPool` should create a PoolMaster and `getPoolMaster` should return it", function* (done) { + try { + r = r.createPools(options); + assert(r.getPoolMaster(config)); + assert.equal(r.getPoolMaster().getPools().length, 1) + done(); + } + catch(e) { + done(e); + } +}); + +//TODO try to make this tests a little more deterministic +It("`run` should work without a connection if a pool exists", function* (done) { + try { + result = yield r.expr(1).run() + assert.equal(result, 1); + + assert(r.getPool().getAvailableLength() >= 2); // This can be 2 because r.expr(1) may be run BEFORE a connection in the buffer is available + assert(r.getPool().getAvailableLength() <= r.getPool().getLength()) + done() + } + catch(e) { + done(e); + } +}); +It("The pool should keep a buffer", function* (done) { + try { + result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] + assert.deepEqual(result, [1,1,1,1,1]); + assert(r.getPool(0).getLength() >= options.buffer+result.length); + + setTimeout( function() { + assert(r.getPool(0).getAvailableLength() >= result.length) // The connections created for the buffer may not be available yet + assert.equal(r.getPool(0).getLength(), r.getPool(0).getLength()) + done(); + }, 500) + } + catch(e) { + done(e); + } +}); +It("A noreply query should release the connection", function* (done) { + try { + var numConnections = r.getPool(0).getLength(); + yield r.expr(1).run({noreply: true}) + assert.equal(r.getPool(0).getLength(), numConnections); + done(); + } + catch(e) { + console.log(e) + done(e); + } +}); +It("The pool shouldn't have more than `options.max` connections", function* (done) { + try { + result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] + assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) + assert.equal(r.getPool(0).getLength(), options.max) + + setTimeout( function() { + assert.equal(r.getPool(0).getAvailableLength(), options.max) + assert.equal(r.getPool(0).getAvailableLength(), r.getPool(0).getLength()) + done() + }, 500) + } + catch(e) { + done(e); + } +}); + +It("Init for `pool.js`", function* (done) { + try { + dbName = uuid(); + tableName = uuid(); + + result = yield r.dbCreate(dbName).run(); + assert.equal(result.dbs_created, 1); + + result = yield r.db(dbName).tableCreate(tableName).run(); + assert.equal(result.tables_created, 1); + + result = yield r.db(dbName).table(tableName).insert(eval('['+new Array(10000).join('{}, ')+'{}]')).run(); + assert.equal(result.inserted, 10000); + pks = result.generated_keys; + + done(); + } + catch(e) { + done(e); + } +}) +It("Updating data to make it heavier", function* (done) { + try { + //Making bigger documents to retrieve multiple batches + var result = yield r.db(dbName).table(tableName).update({ + "foo": uuid(), + "fooo": uuid(), + "foooo": uuid(), + "fooooo": uuid(), + "foooooo": uuid(), + "fooooooo": uuid(), + "foooooooo": uuid(), + "fooooooooo": uuid(), + "foooooooooo": uuid(), + date: r.now() + }).run(); + done(); + } + catch(e) { + done(e); + } +}) + + + +It("The pool should release a connection only when the cursor has fetch everything or get closed", function* (done) { + try { + result = yield [r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true}),r.db(dbName).table(tableName).run({cursor: true})]; + assert.equal(result.length, 10); + assert.equal(r.getPool(0).getAvailableLength(), 0); + yield result[0].toArray(); + assert.equal(r.getPool(0).getAvailableLength(), 1); + yield result[1].toArray(); + assert.equal(r.getPool(0).getAvailableLength(), 2); + yield result[2].close(); + assert.equal(r.getPool(0).getAvailableLength(), 3); + yield [result[3].close(), result[4].close(), result[5].close(), result[6].close(), result[7].close(), result[8].close(), result[9].close()] + done(); + } + catch(e) { + done(e); + } +}); + +It("The pool should shrink if a connection is not used for some time", function* (done) { + try{ + r.getPool(0).setOptions({timeoutGb: 100}); + + result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] + + assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) + + setTimeout(function() { + assert.equal(r.getPool(0).getAvailableLength(), options.buffer) + assert.equal(r.getPool(0).getLength(), options.buffer) + done() + },400) + } + catch(e) { + done(e); + } +}); + +It("`poolMaster.drain` should eventually remove all the connections", function* (done) { + try{ + yield r.getPoolMaster().drain(); + + assert.equal(r.getPool(0).getAvailableLength(), 0) + assert.equal(r.getPool(0).getLength(), 0) + + done() + } + catch(e) { + done(e); + } +}); +It("If the pool cannot create a connection, it should reject queries", function* (done) { + try { + var r = require(__dirname+'/../lib')({host: "notarealhost", buffer: 1, max: 2, silent: true}); + yield r.expr(1).run() + done(new Error("Was expecting an error")); + } + catch(e) { + if (e.message === "None of the pools have an opened connection and failed to open a new one.") { + done() + } + else { + done(e); + } + } +}); +It("If the pool cannot create a connection, it should reject queries - timeout", function* (done) { + try { + var r = require(__dirname+'/../lib')({host: "notarealhost", buffer: 1, max: 2, silent: true}); + yield new Promise(function(resolve, reject) { setTimeout(resolve, 1000) }); + yield r.expr(1).run() + done(new Error("Was expecting an error")); + } + catch(e) { + if (e.message === "None of the pools have an opened connection and failed to open a new one.") { + done() + } + else { + done(e); + } + } +}); + + +It("If the pool is drained, it should reject queries", function* (done) { + try { + var r = require(__dirname+'/../lib')({buffer: 1, max: 2, silent: true}); + + r.getPool(0).drain(); + + var result = yield r.expr(1).run(); + done(new Error("Was expecting an error")); + } + catch(e) { + if (e.message === "None of the pools have an opened connection and failed to open a new one.") { + done() + } + else { + done(e); + } + } +}); + +It("`drain` should work in case of failures", function* (done) { + try { + r = r.createPools({ + port: 80, // non valid port + silent: true, + timeoutError: 100 + }); + var pool = r.getPool(0); + // Sleep 1 sec + yield new Promise(function(resolve, reject) { setTimeout(resolve, 150) }); + pool.drain(); + + // timeoutReconnect should have been canceled + assert.equal(pool.timeoutReconnect, null); + pool.options.silent = false; + yield new Promise(function(resolve, reject) { setTimeout(resolve, 1000) }); + done(); + } + catch(e) { + done(e); + } +}); + + +/* +// This doesn't work anymore because since the JSON protocol was introduced. +It("The pool should remove a connection if it errored", function* (done) { + try{ + r.getPool(0).setOptions({timeoutGb: 60*60*1000}); + + result = yield [r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run(), r.expr(1).run()] + + assert.deepEqual(result, [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]) + + // This query will make the error return an error -1 + result = yield r.expr(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1).add(1) + .run() + + + } + catch(e) { + if ((true) || (e.message === "Client is buggy (failed to deserialize protobuf)")) { + // We expect the connection that errored to get closed in the next second + setTimeout(function() { + assert.equal(r.getPool().getAvailableLength(), options.max-1) + assert.equal(r.getPool().getLength(), options.max-1) + done() + }, 1000) + } + else { + done(e); + } + + } +}); +*/ diff --git a/test/stream.js b/test/stream.js index 2ec333c..13a0701 100644 --- a/test/stream.js +++ b/test/stream.js @@ -12,7 +12,6 @@ var dbName, tableName, tableName2, stream, result, pks, feed; var numDocs = 100; // Number of documents in the "big table" used to test the SUCCESS_PARTIAL var smallNumDocs = 5; // Number of documents in the "small table" - It("Init for `stream.js`", function* (done) { try { dbName = uuid(); @@ -282,7 +281,7 @@ It("Test read", function* (done) { }) It("Import with stream as default", function* (done) { - var r1 = require('../lib')({stream: true, host: config.host, port: config.port, authKey: config.authKey, buffer: config.buffer, max: config.max}); + var r1 = require('../lib')({stream: true, host: config.host, port: config.port, authKey: config.authKey, buffer: config.buffer, max: config.max, auto: false}); var i=0; try { stream = yield r1.db(dbName).table(tableName).run(); @@ -340,7 +339,7 @@ It("toStream - with grouped data", function* (done) { }) It("pipe should work with a writable stream - 200-200", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); r1.db(dbName).table(tableName).toStream({highWaterMark: 200}) .pipe(r1.db(dbName).table(dumpTable).toStream({writable: true, highWaterMark: 200})) @@ -354,13 +353,15 @@ It("pipe should work with a writable stream - 200-200", function* (done) { } return r1.db(dbName).table(dumpTable).delete() }).then(function() { - r1.getPool().drain(); - done(); + r1.getPool(0).drain(); + }).then(function() { + setTimeout(done, 1000); + //done(); }).error(done); }); }) It("pipe should work with a writable stream - 200-20", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); r1.db(dbName).table(tableName).toStream({highWaterMark: 200}) .pipe(r1.db(dbName).table(dumpTable).toStream({writable: true, highWaterMark: 20})) @@ -374,13 +375,13 @@ It("pipe should work with a writable stream - 200-20", function* (done) { } return r1.db(dbName).table(dumpTable).delete() }).then(function() { - r1.getPool().drain(); + r1.getPool(0).drain(); done(); }).error(done); }); }) It("pipe should work with a writable stream - 20-200", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); r1.db(dbName).table(tableName).toStream({highWaterMark: 20}) .pipe(r1.db(dbName).table(dumpTable).toStream({writable: true, highWaterMark: 200})) @@ -394,13 +395,13 @@ It("pipe should work with a writable stream - 20-200", function* (done) { } return r1.db(dbName).table(dumpTable).delete() }).then(function() { - r1.getPool().drain(); + r1.getPool(0).drain(); done(); }).error(done); }); }) It("pipe should work with a writable stream - 50-50", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); r1.db(dbName).table(tableName).toStream({highWaterMark: 50}) .pipe(r1.db(dbName).table(dumpTable).toStream({writable: true, highWaterMark: 50})) @@ -415,7 +416,7 @@ It("pipe should work with a writable stream - 50-50", function* (done) { } return r1.db(dbName).table(dumpTable).delete() }).then(function() { - r1.getPool().drain(); + r1.getPool(0).drain(); done(); }).error(function(err) { console.log(err); @@ -424,7 +425,7 @@ It("pipe should work with a writable stream - 50-50", function* (done) { }); }) It("toStream((writable: true}) should handle options", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); var stream = r1.db(dbName).table(dumpTable).toStream({writable: true, highWaterMark: 50, conflict: 'replace'}); stream.write({id: 1, foo: 1}); @@ -438,7 +439,7 @@ It("toStream((writable: true}) should handle options", function* (done) { assert.deepEqual(result, {id: 1, foo: 3}); return r1.db(dbName).table(dumpTable).delete(); }).then(function(result) { - r1.getPool().drain(); + r1.getPool(0).drain(); done(); }).error(done); }); @@ -484,13 +485,13 @@ It("test pipe all streams", function* (done) { return r.db(dbName).table(dumpTable).delete(); }).then(function(result) { done(); - r.getPool().drain(); + r.getPool(0).drain(); }); }); }) It("toStream({writable: true}) should throw on something else than a table", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); try { r.expr(dumpTable).toStream({writable: true}); @@ -502,7 +503,7 @@ It("toStream({writable: true}) should throw on something else than a table", fun }) It("toStream({transform: true}) should throw on something else than a table", function* (done) { - var r1 = require('../lib')({buffer:1, max: 2}); + var r1 = require('../lib')({buffer:1, max: 2, auto: false}); try { r.expr(dumpTable).toStream({transform: true}); diff --git a/test/util/common.js b/test/util/common.js index 8371b49..d73b815 100644 --- a/test/util/common.js +++ b/test/util/common.js @@ -17,7 +17,12 @@ function It(testName, generatorFn) { Promise.coroutine(generatorFn)(done); }) } - +function sleep(timer) { + return new Promise(function(resolve, reject) { + setTimeout(resolve, timer); + }); +} module.exports.uuid = uuid module.exports.It = It +module.exports.sleep = sleep diff --git a/test/util/fake_server/README.md b/test/util/fake_server/README.md new file mode 100644 index 0000000..759e02d --- /dev/null +++ b/test/util/fake_server/README.md @@ -0,0 +1,5 @@ +This is a fork of [reqlite](https://github.com/neumino/github). + +The project was not totally complete, and this fork includes hacks to: +- simulate network/query latency in a predicatable way +- simulate new/removed servers in `servers_status`. diff --git a/test/util/fake_server/database.js b/test/util/fake_server/database.js new file mode 100644 index 0000000..06fd779 --- /dev/null +++ b/test/util/fake_server/database.js @@ -0,0 +1,21 @@ +var Table = require(__dirname+"/table.js"); + +function Database(name) { + this.name = name; + this.tables = {}; +} + +Database.prototype.table = function(name) { + return this.tables[name]; +} +Database.prototype.tableDrop = function(name) { + delete this.tables[name]; +} +Database.prototype.tableCreate = function(name) { + this.tables[name] = new Table(name) +} +Database.prototype.typeOf = function() { + return "DB"; +} + +module.exports = Database; diff --git a/test/util/fake_server/document.js b/test/util/fake_server/document.js new file mode 100644 index 0000000..acd1696 --- /dev/null +++ b/test/util/fake_server/document.js @@ -0,0 +1,133 @@ +var helper = require(__dirname+"/helper.js"); + +function Document(doc, table) { + this.doc = doc; + this.table = table; +} + +Document.prototype.typeOf = function() { + return "SELECTION"; +} +Document.prototype.update = function(newValue, options, query) { + options = options || {}; + + var result = helper.writeResult(); + var primaryKey = this.table.options.primaryKey; + + // Update context + if (newValue[0] === 69) { // newValue is a FUNC term + var varId = newValue[1][0][1][0]; // 0 to select the array, 1 to select the first element + query.context[varId] = this; + } + var updateValue = query.evaluate(newValue) + // Clean context + if (newValue[0] === 69) { + delete query.context[varId] + } + + + // TODO Refactor with replace + if ((updateValue[primaryKey] !== undefined) + && (this.doc[primaryKey] !== updateValue[primaryKey])) { + result.errors++; + if(result.first_error === undefined) { + result.first_error = "Primary key `id` cannot be changed(`"+ + JSON.stringify(updateValue, null, 2)+"` -> `"+ + JSON.stringify(this.doc, null, 2)+"`)" + } + } + else { + if (options.return_vals === true) { + result.old_val = helper.deepCopy(this.doc); + } + var changed = helper._merge(this.doc, updateValue) + + if (options.return_vals === true) { + result.new_val = helper.deepCopy(this.doc); + } + + if (changed === true) { + result.replaced++; + } + else { + result.unchagned++; + } + } + return result; +} + +Document.prototype.replace = function(newValue, options, query) { + options = options || {}; + + var result = helper.writeResult(); + var primaryKey = this.table.options.primaryKey; + + // Update context + if (newValue[0] === 69) { // newValue is a FUNC term + var varId = newValue[1][0][1][0]; // 0 to select the array, 1 to select the first element + query.context[varId] = this; + } + var replaceValue = query.evaluate(newValue) + // Clean context + if (newValue[0] === 69) { + delete query.context[varId] + } + + + + if ((replaceValue[primaryKey] !== undefined) + && (this.doc[primaryKey] !== replaceValue[primaryKey])) { + result.errors++; + if(result.first_error === undefined) { + result.first_error = "Primary key `id` cannot be changed(`"+ + JSON.stringify(replaceValue, null, 2)+"` -> `"+ + JSON.stringify(this.doc, null, 2)+"`)" + } + } + else { + if (options.return_vals === true) { + result.old_val = helper.deepCopy(this.doc); + } + var changed = helper._replace(this.doc, replaceValue) + + if (options.return_vals === true) { + result.new_val = helper.deepCopy(this.doc); + } + + if (changed === true) { + result.replaced++; + } + else { + result.unchanged++; + } + } + return result; +} + +Document.prototype.delete = function() { + var result = helper.writeResult(); + var primaryKey = this.table.options.primaryKey; + + return this.table._delete(this.doc); + +} + +Document.prototype.toDatum = function() { + var result = {}; + for(var key in this.doc) { + if (typeof this.doc[key].toDatum === "function") { + result[key] = this.doc[key].toDatum(); + } + else { + result[key] = helper.toDatum(this.doc[key]); + } + } + return result; +} + +Document.prototype.getField = function(field) { + return this.doc[field]; +} + + +module.exports = Document; diff --git a/test/util/fake_server/error.js b/test/util/fake_server/error.js new file mode 100644 index 0000000..096dd90 --- /dev/null +++ b/test/util/fake_server/error.js @@ -0,0 +1,39 @@ +var protodef = require(__dirname+"/protodef.js"); + +function ReqlRuntimeError(msg, frames) { + Error.captureStackTrace(this, ReqlRuntimeError) + this.type = protodef.Response.ResponseType.RUNTIME_ERROR; + this.message = msg; + this.frames = frames || []; +} +ReqlRuntimeError.prototype = new Error(); +//ReqlRuntimeError.prototype.name = "ReqlRuntimeError" + + +function ReqlClientError(msg, frames) { + Error.captureStackTrace(this, ReqlClientError) + this.type = protodef.Response.ResponseType.CLIENT_ERROR; + this.message = msg; + this.frames = frames || []; +} +ReqlClientError.prototype = new Error(); +//ReqlClientError.prototype.name = "ReqlClientError" + +function ReqlCompileError(msg, frames) { + Error.captureStackTrace(this, ReqlCompileError) + this.type = protodef.Response.ResponseType.COMPILE_ERROR; + this.message = msg; + this.frames = frames || []; +} +ReqlCompileError.prototype = new Error(); +ReqlCompileError.prototype.name = "ReqlCompileError" + + +Err = {}; +Err.ReqlRuntimeError = ReqlRuntimeError; +Err.ReqlCompileError = ReqlCompileError; +Err.ReqLClientError = ReqlClientError; + + + +module.exports = Err; diff --git a/test/util/fake_server/group.js b/test/util/fake_server/group.js new file mode 100644 index 0000000..99966e7 --- /dev/null +++ b/test/util/fake_server/group.js @@ -0,0 +1,74 @@ +var helper = require(__dirname+"/helper.js"); +var Sequence = require(__dirname+"/sequence.js"); +var util = require('util'); + +function Group(groups) { + this.groups = groups || []; // {group : , reduction: } + // this.type? +} + +// Import methods from Sequence +var keys = Object.keys(Sequence.prototype); +for(var i=0; i 0) && (this.groups[0].reduction instanceof Sequence)) { + return "GROUPED_STREAM"; + } + else { + return "GROUPED_DATA"; + } +} + +Group.prototype.toDatum = function() { + var result = []; + for(var i=0; i"; + } + else if (helper.isPlainObject(value)) { + return "OBJECT"; + } + throw new Error.ReqlRuntimeError("Server is buggy, unknown type", query.frames) + +} +helper.assertType = function assertType(value, type, query) { + var Sequence = require(__dirname+"/sequence.js"); + var typeValue; + + //TODO Add group? + if (value instanceof Sequence) { + typeValue = "ARRAY"; + } + else if (value === null) { + typeValue = "NULL" + } + else if (helper.isDate(value)) { + typeValue = "PTYPE