From 36890398e0d09af83613be68aa5811943dc03fae Mon Sep 17 00:00:00 2001 From: Agnes Lin Date: Wed, 13 Nov 2019 13:43:46 -0500 Subject: [PATCH] test: setup couchdb docker image for travis --- .travis.yml | 16 +- .../repository-cloudant/package-lock.json | 6 +- acceptance/repository-cloudant/package.json | 4 +- docker.setup.js | 310 ++++++++++++++++++ package-lock.json | 163 +++++++++ package.json | 9 +- 6 files changed, 488 insertions(+), 20 deletions(-) create mode 100644 docker.setup.js diff --git a/.travis.yml b/.travis.yml index 180cb9172967..236c7b010f7e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -84,22 +84,12 @@ matrix: os: linux env: - TASK=test-repository-cloudant - - CLOUDANT_DATABASE=testdb - - CLOUDANT_USER=admin - - CLOUDANT_PASSWORD=pass - - CLOUDANT_PORT=5984 - services: - - couchdb - before_script: - - curl http://localhost:5984 - - curl -X PUT http://localhost:5984/testdb - - curl -X PUT http://localhost:5984/_node/couchdb@localhost/_config/admins/admin -d '"pass"' - - curl -s -X PUT http://localhost:5984/_config/admins/admin -d '"pass"' - + install: + - npm ci && npm run docker:setup && source cloudant-config.sh && pwd script: - npm run postinstall -- --scope "@loopback/test-repository-cloudant" --include-dependencies - npm run build -- --scope "@loopback/test-repository-cloudant" --include-dependencies - - cd acceptance/repository-cloudant && npm run mocha + - source cloudant-config.sh && cd acceptance/repository-cloudant && npm run mocha branches: only: diff --git a/acceptance/repository-cloudant/package-lock.json b/acceptance/repository-cloudant/package-lock.json index 938e72668dd9..34a3162c8672 100644 --- a/acceptance/repository-cloudant/package-lock.json +++ b/acceptance/repository-cloudant/package-lock.json @@ -5,9 +5,9 @@ "requires": true, "dependencies": { "@types/node": { - "version": "10.14.22", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.22.tgz", - "integrity": "sha512-9taxKC944BqoTVjE+UT3pQH0nHZlTvITwfsOZqyc+R3sfJuxaTtxWjfn1K2UlxyPcKHf0rnaXcVFrS9F9vf0bw==", + "version": "10.17.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.5.tgz", + "integrity": "sha512-RElZIr/7JreF1eY6oD5RF3kpmdcreuQPjg5ri4oQ5g9sq7YWU8HkfB3eH8GwAwxf5OaCh0VPi7r4N/yoTGelrA==", "dev": true }, "accept-language": { diff --git a/acceptance/repository-cloudant/package.json b/acceptance/repository-cloudant/package.json index b5fbc7a8291b..60356e6c09d6 100644 --- a/acceptance/repository-cloudant/package.json +++ b/acceptance/repository-cloudant/package.json @@ -21,8 +21,8 @@ "@loopback/build": "^1.7.1", "@loopback/eslint-config": "^4.1.2", "@loopback/repository": "^1.15.2", - "@loopback/repository-tests": "^0.5.2", - "@types/node": "^10.14.22", + "@loopback/repository-tests": "^0.6.1", + "@types/node": "^10.17.5", "loopback-connector-cloudant": "2.3.2" }, "files": [ diff --git a/docker.setup.js b/docker.setup.js new file mode 100644 index 000000000000..c4552b671bc4 --- /dev/null +++ b/docker.setup.js @@ -0,0 +1,310 @@ +// Copyright IBM Corp. 2017,2019. All Rights Reserved. +// Node module: loopback-connector-cloudant +// This file is licensed under the Artistic License 2.0. +// License text available at https://opensource.org/licenses/Artistic-2.0 + +'use strict'; + +const _ = require('lodash'); +const async = require('async'); +const spawn = require('child_process').spawn; +const docker = new require('dockerode')(); +const fmt = require('util').format; +const http = require('http'); +const ms = require('ms'); +const fs = require('fs-extra'); + +// // we don't pass any node flags, so we can call _mocha instead the wrapper +// const mochaBin = require.resolve('mocha/bin/_mocha'); + +process.env.CLOUDANT_DATABASE = 'testdb'; +process.env.CLOUDANT_PASSWORD = 'pass'; +process.env.CLOUDANT_USERNAME = 'admin'; + +// these are placeholders. They get set dynamically based on what IP and port +// get assigned by docker. +process.env.CLOUDANT_PORT = 'TBD'; +process.env.CLOUDANT_HOST = 'TBD'; +process.env.CLOUDANT_URL = 'TBD'; + +const CONNECT_RETRIES = 30; +const CONNECT_DELAY = ms('5s'); + +let containerToDelete = null; + +async.waterfall( + [ + dockerStart('ibmcom/couchdb3:latest'), + sleep(ms('2s')), + setCloudantEnv, + waitFor('/_all_dbs'), + createAdmin(), + createDB('testdb'), + listUser(), + exportENV(), + ], + function(testErr) { + if (testErr) { + console.error('error running tests:', testErr); + process.exit(1); + } + }, +); + +function sleep(n) { + return function delayedPassThrough() { + const args = [].slice.call(arguments); + // last argument is the callback + const next = args.pop(); + // prepend `null` to indicate no error + args.unshift(null); + setTimeout(function() { + next.apply(null, args); + }, n); + }; +} + +function dockerStart(imgName) { + return function pullAndStart(next) { + console.log('pulling image: %s', imgName); + docker.pull(imgName, function(err, stream) { + docker.modem.followProgress(stream, function(err, output) { + if (err) { + return next(err); + } + console.log('starting container from image: %s', imgName); + docker.createContainer( + { + Image: imgName, + HostConfig: { + PublishAllPorts: true, + }, + }, + function(err, container) { + console.log( + 'recording container for later cleanup: ', + container.id, + ); + containerToDelete = container; + if (err) { + return next(err); + } + container.start(function(err, data) { + next(err, container); + }); + }, + ); + }); + }); + }; +} + +function setCloudantEnv(container, next) { + container.inspect(function(err, c) { + // if swarm, Node.Ip will be set to actual node's IP + // if not swarm, but remote docker, use docker host's IP + // if local docker, use localhost + const host = _.get(c, 'Node.IP', _.get(docker, 'modem.host', '127.0.0.1')); + // couchdb uses TCP/IP port 5984 + // container's port 5984 is dynamically mapped to an external port + const port = _.get(c, [ + 'NetworkSettings', + 'Ports', + '5984/tcp', + '0', + 'HostPort', + ]); + process.env.CLOUDANT_PORT = port; + process.env.CLOUDANT_HOST = host; + const usr = process.env.CLOUDANT_USERNAME; + const pass = process.env.CLOUDANT_PASSWORD; + process.env.CLOUDANT_URL = + 'http://' + usr + ':' + pass + '@' + host + ':' + port; + console.log( + 'env:', + _.pick(process.env, [ + 'CLOUDANT_URL', + 'CLOUDANT_HOST', + 'CLOUDANT_PORT', + 'CLOUDANT_USERNAME', + 'CLOUDANT_PASSWORD', + 'CLOUDANT_DATABASE', + ]), + ); + next(null, container); + }); +} + +function waitFor(path) { + return function waitForPath(container, next) { + const opts = { + host: process.env.CLOUDANT_HOST, + port: process.env.CLOUDANT_PORT, + path: path, + }; + + console.log(`waiting for instance to respond: ${opts}`); + return ping(null, CONNECT_RETRIES); + + function ping(err, tries) { + console.log('ping (%d/%d)', CONNECT_RETRIES - tries, CONNECT_RETRIES); + if (tries < 1) { + next(err || new Error('failed to contact Cloudant')); + } + http + .get(opts, function(res) { + res.pipe(devNull()); + res.on('error', tryAgain); + res.on('end', function() { + if (res.statusCode === 200) { + setImmediate(next, null, container); + } else { + tryAgain(); + } + }); + }) + .on('error', tryAgain); + function tryAgain(err) { + setTimeout(ping, CONNECT_DELAY, err, tries - 1); + } + } + }; +} + +function createAdmin() { + return function createAdminUser(container, next) { + const data = '"pass"'; + const uri = + '/_node/couchdb@127.0.0.1/_config/admins/' + + process.env.CLOUDANT_USERNAME; + const opts = { + method: 'PUT', + path: uri, + header: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + host: process.env.CLOUDANT_HOST, + port: process.env.CLOUDANT_PORT, + body: data, + }; + + const req = http.request(opts, function(res) { + res.pipe(devNull()); + res.on('error', next); + res.on('end', function() { + setImmediate(next, null, container); + }); + }); + req.write(data); + req.end(); + }; +} + +function createDB(db) { + return function create(container, next) { + const opts = { + method: 'PUT', + path: '/' + db, + host: process.env.CLOUDANT_HOST, + port: process.env.CLOUDANT_PORT, + auth: process.env.CLOUDANT_USERNAME + ':' + process.env.CLOUDANT_PASSWORD, + }; + console.log('creating db: %j', db); + http + .request(opts, function(res) { + res.pipe(devNull()); + res.on('error', next); + res.on('end', function() { + setImmediate(next, null, container); + }); + }) + .on('error', next) + .end(); + }; +} + +function listUser() { + return function printUsers(container, next) { + const path = '/_node/couchdb@127.0.0.1/_config/admins'; + const opts = { + method: 'GET', + path: path, + host: process.env.CLOUDANT_HOST, + port: process.env.CLOUDANT_PORT, + auth: process.env.CLOUDANT_USERNAME + ':' + process.env.CLOUDANT_PASSWORD, + }; + // console.log('creating db: %j', db); + http + .request(opts, function(res) { + res.pipe(devNull()); + res.on('error', next); + + res.setEncoding('utf8'); + let rawData = ''; + res.on('data', chunk => { + rawData += chunk; + }); + + res.on('end', function() { + try { + const parsedData = JSON.parse(rawData); + console.log(parsedData); + } catch (e) { + console.error(e.message); + } + setImmediate(next, null, container); + }); + }) + .on('error', next) + .end(); + }; +} + +function exportENV() { + return function createENVFile(container, next) { + const content = + "export CLOUDANT_URL='" + + process.env.CLOUDANT_URL + + "'\n" + + "export CLOUDANT_DATABASE='" + + process.env.CLOUDANT_DATABASE + + "'\n" + + "export CLOUDANT_USER='" + + process.env.CLOUDANT_USERNAME + + "'\n" + + "export CLOUDANT_PASSWORD='" + + process.env.CLOUDANT_PASSWORD + + "'\n" + + 'export CLOUDANT_PORT=' + + process.env.CLOUDANT_PORT + + '\n' + + "export CLOUDANT_HOST='" + + process.env.CLOUDANT_HOST + + "'"; + console.log('current directory: ' + __dirname); + fs.writeFileSync('cloudant-config.sh', content, 'utf8'); + next(); + }; +} + +// clean up any previous containers +function dockerCleanup(next) { + if (containerToDelete) { + console.log('cleaning up container: %s', containerToDelete.id); + containerToDelete.remove({force: true}, function(err) { + next(err); + }); + } else { + setImmediate(next); + } +} + +// A Writable Stream that just consumes a stream. Useful for draining readable +// streams so that they 'end' properly, like sometimes-empty http responses. +function devNull() { + return new require('stream').Writable({ + write: function(_chunk, _encoding, cb) { + return cb(null); + }, + }); +} diff --git a/package-lock.json b/package-lock.json index 587ee03a9cf9..6dc12a1e1fec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1810,6 +1810,12 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "async": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.1.0.tgz", + "integrity": "sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ==", + "dev": true + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -1945,6 +1951,16 @@ "integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==", "dev": true }, + "bl": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", + "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", + "dev": true, + "requires": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, "bluebird": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.1.tgz", @@ -1996,6 +2012,28 @@ "integrity": "sha1-M3dm2hWAEhD92VbCLpxokaudAzc=", "dev": true }, + "buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dev": true, + "requires": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "dev": true + }, + "buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", + "dev": true + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -3080,6 +3118,74 @@ "path-type": "^3.0.0" } }, + "docker-modem": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-1.0.9.tgz", + "integrity": "sha512-lVjqCSCIAUDZPAZIeyM125HXfNvOmYYInciphNrLrylUtKyW66meAjSPXWchKVzoIYZx69TPnAepVSSkeawoIw==", + "dev": true, + "requires": { + "JSONStream": "1.3.2", + "debug": "^3.2.6", + "readable-stream": "~1.0.26-4", + "split-ca": "^1.0.0" + }, + "dependencies": { + "JSONStream": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz", + "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=", + "dev": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "dev": true + }, + "readable-stream": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", + "dev": true + } + } + }, + "dockerode": { + "version": "2.5.8", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-2.5.8.tgz", + "integrity": "sha512-+7iOUYBeDTScmOmQqpUYQaE7F4vvIt6+gIZNHWhqAQEI887tiPFB9OvXI/HzQYqfUNvukMK+9myLW63oTJPZpw==", + "dev": true, + "requires": { + "concat-stream": "~1.6.2", + "docker-modem": "^1.0.8", + "tar-fs": "~1.16.3" + } + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -3901,6 +4007,12 @@ "readable-stream": "^2.0.0" } }, + "fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -7226,6 +7338,12 @@ "through": "2" } }, + "split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha1-bIOv82kvphJW4M0ZfgXp3hV2kaY=", + "dev": true + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", @@ -7470,6 +7588,45 @@ "yallist": "^3.0.3" } }, + "tar-fs": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", + "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", + "dev": true, + "requires": { + "chownr": "^1.0.1", + "mkdirp": "^0.5.1", + "pump": "^1.0.0", + "tar-stream": "^1.1.2" + }, + "dependencies": { + "pump": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", + "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, + "tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dev": true, + "requires": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + } + }, "temp-dir": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-1.0.0.tgz", @@ -7545,6 +7702,12 @@ "os-tmpdir": "~1.0.2" } }, + "to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", + "dev": true + }, "to-object-path": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", diff --git a/package.json b/package.json index c3dbb08a230f..58063d3b5905 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,11 @@ "fs-extra": "^8.1.0", "husky": "^3.0.9", "lerna": "^3.18.4", - "typescript": "~3.7.2" + "typescript": "~3.7.2", + "async": "^3.1.0", + "debug": "^4.1.1", + "lodash": "^4.17.11", + "dockerode": "^2.4.3" }, "scripts": { "postinstall": "lerna bootstrap", @@ -62,7 +66,8 @@ "docs:prepare": "./docs/bin/build-preview-site.sh", "docs:start": "cd docs/_preview && bundle exec jekyll serve --no-w --i", "mocha": "node packages/build/bin/run-mocha \"packages/*/dist/__tests__/**/*.js\" \"extensions/*/dist/__tests__/**/*.js\" \"examples/*/dist/__tests__/**/*.js\" \"packages/cli/test/**/*.js\" \"packages/build/test/*/*.js\"", - "posttest": "npm run lint" + "posttest": "npm run lint", + "docker:setup": "node ./docker.setup.js" }, "config": { "commitizen": {