diff --git a/doc/cli/npm-archive.md b/doc/cli/npm-archive.md new file mode 100644 index 0000000000000..389402d21d171 --- /dev/null +++ b/doc/cli/npm-archive.md @@ -0,0 +1,85 @@ +npm-archive(1) -- Project-local dependency tarball archive +=================================== + +## SYNOPSIS + + npm archive + +## EXAMPLE + +Make sure you have a package-lock and an up-to-date install: + +``` +$ cd ./my/npm/project +$ npm install +added 154 packages in 10s +$ ls | grep package-lock +``` + +Run `npm archive` in that project + +``` +$ npm archive +added 1964 packages in 4.103s +``` + +Commit the newly-created `archived-packages/` directory and the modified `package-lock.json` + +``` +$ git add package-lock.json archived-packages/ +$ git commit -m 'misc: committing dependency archive' +``` + +Add a dependency as usual -- its archive will be automatically managed. + +``` +$ npm i aubergine +added 1 package from 1 contributor in 5s +$ git status + M package-lock.json + M package.json +?? archived-packages/aubergine-1.0.1-46c5742af.tar +$ git add archived-packages package-lock.json package.json +$ git commit -m 'deps: aubergine@1.0.1' +``` + +The inverse happens when a package is removed. + +You can then install normally using `npm-ci(1)` or `npm-install(1)`! + +``` +$ npm ci +added 1965 packages in 10.5s +``` + +Finally, you can remove and disable the archive, restoring `package-lock.json` its normal state, by using `npm-unarchive(1)`. + +``` +$ npm unarchive + +``` +## DESCRIPTION + +This command generates a committable archive of your project's dependencies. There are several benefits to this: + +1. Offline installs without having to warm up npm's global cache +2. No need for configuring credentials for dependency fetching +3. Much faster installs vs standard CI configurations +4. No need to have a `git` binary present in the system +5. Reduced download duplication across teams + +`npm-archive` works by generating tarballs for your dependencies, unzipping them, and storing them in a directory called `archived-packages/`. It then rewrites your `package-lock.json` (or `npm-shrinkwrap.json`) such that the `resolved` field on those dependencies refers to the path in `archived-packages/`. + +npm will detect these `file:` URLs and extract package data directly from them instead of the registry, git repositories, etc. + +When installing or removing dependencies, npm will look for `archived-packages/` and switch to an "archive mode", which will automatically update archive files and information on every relevant npm operation. Remember to commit the directory, not just `package-lock.json`! + +As an added benefit, `npm-archive` will generate tarballs for all your git dependencies and pre-pack them, meaning npm will not need to invoke the git binary or go through other heavy processes git dependencies go to -- making git deps as fast as registry dependencies when reinstalling from an archive. + +If specific tarballs are removed from the archive, npm will fall back to standard behavior for fetching dependencies: first checking its global cache, then going out and fetching the dependency from its origin. To regenerate the tarball for a package after removing it, just reinstall the package while in archive mode. + +## SEE ALSO + +* npm-unarchive(1) +* npm-package-locks(5) +* npm-ci(1) diff --git a/doc/cli/npm-unarchive.md b/doc/cli/npm-unarchive.md new file mode 100644 index 0000000000000..8cb46a19aae1e --- /dev/null +++ b/doc/cli/npm-unarchive.md @@ -0,0 +1,25 @@ +npm-unarchive(1) -- Restore project to a non-archived state. +=================================== + +## SYNOPSIS + + npm unarchive + +## EXAMPLE + +``` +$ npm unarchive +archive information and tarballs removed +``` +## DESCRIPTION + +This command undoes the work of `npm-archive(1)` by doing the following: + +1. Removes the `archived-packages/` directory. +2. Restores the entires in `package-lock.json` to use non-`file:` resolved URLs and updates their `integrity` fields. +3. Removes `node_modules/` to prevent archive-related changes from affecting future installs. + +## SEE ALSO + +* npm-archive(1) +* npm-package-locks(5) diff --git a/doc/misc/npm-config.md b/doc/misc/npm-config.md index 8d439331845e1..80865cba40bc0 100644 --- a/doc/misc/npm-config.md +++ b/doc/misc/npm-config.md @@ -164,6 +164,18 @@ When "true" submit audit reports alongside `npm install` runs to the default registry and all registries configured for scopes. See the documentation for npm-audit(1) for details on what is submitted. +### archive + +* Default: true +* Type: Boolean + +If false (with `--no-archive`), an existing `archived-packages/` directory +will not be modified on save. + +This flag has no effect if the archive directory does not alredy exist. + +See also npm-archive(1). + ### auth-type * Default: `'legacy'` diff --git a/lib/archive.js b/lib/archive.js new file mode 100644 index 0000000000000..9527236b527d4 --- /dev/null +++ b/lib/archive.js @@ -0,0 +1,63 @@ +'use strict' + +const BB = require('bluebird') + +const MyPrecious = require('libprecious') +const npm = require('./npm.js') +const npmlog = require('npmlog') +const pacoteOpts = require('./config/pacote.js') +const path = require('path') + +const statAsync = BB.promisify(require('fs').stat) + +archive.usage = 'npm archive\nnpm archive restore' + +archive.completion = (cb) => cb(null, []) + +MyPrecious.PreciousConfig.impl(npm.config, { + get: npm.config.get, + set: npm.config.set, + toPacote (moreOpts) { + return pacoteOpts(moreOpts) + } +}) + +module.exports = archive +function archive (args, cb) { + BB.resolve(_archive()).nodeify(cb) +} + +function _archive (args) { + // TODO - is this the right path?... + return statAsync(path.join(npm.prefix, 'archived-packages')) + .catch((err) => { if (err.code !== 'ENOENT') { throw err } }) + .then((stat) => { + const archiveExists = stat && stat.isDirectory() + return new MyPrecious({ + config: npm.config, + log: npmlog + }) + .run() + .then((details) => { + if (!archiveExists) { + npmlog.notice('archive', 'created new package archive as `archived-packages/`. Future installations will prioritize packages in this directory.') + } + const clauses = [] + if (!details.pkgCount && !details.removed) { + clauses.push('done') + } + if (details.pkgCount) { + clauses.push(`archived ${details.pkgCount} package${ + details.pkgCount === 1 ? '' : 's' + }`) + } + if (details.removed) { + clauses.push(`cleaned up ${details.pkgCount} archive${ + details.removed === 1 ? '' : 's' + }`) + } + const time = details.runTime / 1000 + console.error(`${clauses.join(' and ')} in ${time}s`) + }) + }) +} diff --git a/lib/config/cmd-list.js b/lib/config/cmd-list.js index 2069b5ea33ec7..58ca06d24eb8c 100644 --- a/lib/config/cmd-list.js +++ b/lib/config/cmd-list.js @@ -21,6 +21,7 @@ var shorthands = { } var affordances = { + 'arr': 'archive', 'la': 'ls', 'll': 'ls', 'verison': 'version', @@ -52,6 +53,8 @@ var affordances = { // these are filenames in . var cmdList = [ + 'archive', + 'unarchive', 'ci', 'install', 'install-test', diff --git a/lib/config/defaults.js b/lib/config/defaults.js index 5c324239f2adf..c5eba786313c8 100644 --- a/lib/config/defaults.js +++ b/lib/config/defaults.js @@ -110,6 +110,7 @@ Object.defineProperty(exports, 'defaults', {get: function () { 'always-auth': false, also: null, audit: true, + archive: true, 'auth-type': 'legacy', 'bin-links': true, @@ -255,6 +256,7 @@ exports.types = { 'always-auth': Boolean, also: [null, 'dev', 'development'], audit: Boolean, + archive: Boolean, 'auth-type': ['legacy', 'sso', 'saml', 'oauth'], 'bin-links': Boolean, browser: [null, String], diff --git a/lib/install.js b/lib/install.js index 66f85d80a49a2..634bed1bf20e4 100644 --- a/lib/install.js +++ b/lib/install.js @@ -645,9 +645,9 @@ Installer.prototype.saveToDependencies = function (cb) { if (this.failing) return cb() log.silly('install', 'saveToDependencies') if (this.saveOnlyLock) { - saveShrinkwrap(this.idealTree, cb) + return saveShrinkwrap(this.idealTree).nodeify(cb) } else { - saveRequested(this.idealTree, cb) + return saveRequested(this.idealTree).nodeify(cb) } } diff --git a/lib/install/save.js b/lib/install/save.js index 8bafcbfc6b928..1f333643ef09b 100644 --- a/lib/install/save.js +++ b/lib/install/save.js @@ -1,94 +1,117 @@ 'use strict' +const BB = require('bluebird') + const deepSortObject = require('../utils/deep-sort-object.js') const detectIndent = require('detect-indent') const detectNewline = require('detect-newline') const fs = require('graceful-fs') -const iferr = require('iferr') const log = require('npmlog') const moduleName = require('../utils/module-name.js') +const MyPrecious = require('libprecious') const npm = require('../npm.js') +const pacoteOpts = require('../config/pacote.js') const parseJSON = require('../utils/parse-json.js') const path = require('path') const stringifyPackage = require('../utils/stringify-package') const validate = require('aproba') const without = require('lodash.without') -const writeFileAtomic = require('write-file-atomic') +const writeFileAtomic = BB.promisify(require('write-file-atomic')) + +const readFileAsync = BB.promisify(fs.readFile) +const statAsync = BB.promisify(fs.stat) // if the -S|--save option is specified, then write installed packages // as dependencies to a package.json file. -exports.saveRequested = function (tree, andReturn) { - validate('OF', arguments) - savePackageJson(tree, andWarnErrors(andSaveShrinkwrap(tree, andReturn))) -} - -function andSaveShrinkwrap (tree, andReturn) { - validate('OF', arguments) - return function (er) { - validate('E', arguments) - saveShrinkwrap(tree, andWarnErrors(andReturn)) - } +exports.saveRequested = saveRequested +function saveRequested (tree) { + validate('O', arguments) + return savePackageJson(tree) + .catch(warnError) + .then(() => saveShrinkwrap(tree)) + .catch(warnError) } -function andWarnErrors (cb) { - validate('F', arguments) - return function (er) { - if (er) log.warn('saveError', er.message) - arguments[0] = null - cb.apply(null, arguments) - } +function warnError (err) { + log.warn('saveError', err.message) + log.verbose('saveError', err.stack) } exports.saveShrinkwrap = saveShrinkwrap - -function saveShrinkwrap (tree, next) { - validate('OF', arguments) +function saveShrinkwrap (tree) { + validate('O', arguments) if (!npm.config.get('shrinkwrap') || !npm.config.get('package-lock')) { - return next() + return } - require('../shrinkwrap.js').createShrinkwrap(tree, {silent: false}, next) + const createShrinkwrap = require('../shrinkwrap.js').createShrinkwrap + return BB.fromNode((cb) => createShrinkwrap(tree, {silent: false}, cb)) + .then(() => { + if (!npm.config.get('archive')) { + log.silly('saveArchive', 'skipping archive updates') + return + } + log.silly('saveArchive', 'updating current package archive') + return statAsync(path.resolve(tree.path, 'archived-packages')) + .catch((err) => { if (err.code !== 'ENOENT') { throw err } }) + .then((stat) => { + if (stat && stat.isDirectory()) { + npm.config.toPacote = pacoteOpts + const precious = new MyPrecious({ + config: npm.config, + log + }) + return precious.run() + .then((info) => { + if (info.pkgCount) { + log.verbose('saveArchive', `archived ${info.pkgCount} packages`) + } + if (info.removed) { + log.verbose('saveArchive', `removed ${info.removed} archived packages`) + } + }) + } + }) + }) } -function savePackageJson (tree, next) { - validate('OF', arguments) - var saveBundle = npm.config.get('save-bundle') +function savePackageJson (tree) { + validate('O', arguments) + const saveBundle = npm.config.get('save-bundle') // each item in the tree is a top-level thing that should be saved // to the package.json file. // The relevant tree shape is { : {what:} } - var saveTarget = path.resolve(tree.path, 'package.json') + const saveTarget = path.resolve(tree.path, 'package.json') // don't use readJson, because we don't want to do all the other // tricky npm-specific stuff that's in there. - fs.readFile(saveTarget, 'utf8', iferr(next, function (packagejson) { + return readFileAsync(saveTarget, 'utf8') + .then((packagejson) => { const indent = detectIndent(packagejson).indent const newline = detectNewline(packagejson) - try { - tree.package = parseJSON(packagejson) - } catch (ex) { - return next(ex) - } + tree.package = parseJSON(packagejson) // If we're saving bundled deps, normalize the key before we start + let bundle if (saveBundle) { - var bundle = tree.package.bundleDependencies || tree.package.bundledDependencies + bundle = tree.package.bundleDependencies || tree.package.bundledDependencies delete tree.package.bundledDependencies if (!Array.isArray(bundle)) bundle = [] } - var toSave = getThingsToSave(tree) - var toRemove = getThingsToRemove(tree) - var savingTo = {} - toSave.forEach(function (pkg) { if (pkg.save) savingTo[pkg.save] = true }) - toRemove.forEach(function (pkg) { if (pkg.save) savingTo[pkg.save] = true }) + const toSave = getThingsToSave(tree) + const toRemove = getThingsToRemove(tree) + const savingTo = {} + toSave.forEach((pkg) => { if (pkg.save) savingTo[pkg.save] = true }) + toRemove.forEach((pkg) => { if (pkg.save) savingTo[pkg.save] = true }) - Object.keys(savingTo).forEach(function (save) { + Object.keys(savingTo).forEach((save) => { if (!tree.package[save]) tree.package[save] = {} }) log.verbose('saving', toSave) const types = ['dependencies', 'devDependencies', 'optionalDependencies'] - toSave.forEach(function (pkg) { + toSave.forEach((pkg) => { if (pkg.save) tree.package[pkg.save][pkg.name] = pkg.spec const movedFrom = [] for (let saveType of types) { @@ -110,14 +133,14 @@ function savePackageJson (tree, next) { } }) - toRemove.forEach(function (pkg) { + toRemove.forEach((pkg) => { if (pkg.save) delete tree.package[pkg.save][pkg.name] if (saveBundle) { bundle = without(bundle, pkg.name) } }) - Object.keys(savingTo).forEach(function (key) { + Object.keys(savingTo).forEach((key) => { tree.package[key] = deepSortObject(tree.package[key]) }) if (saveBundle) { @@ -127,11 +150,11 @@ function savePackageJson (tree, next) { var json = stringifyPackage(tree.package, indent, newline) if (json === packagejson) { log.verbose('shrinkwrap', 'skipping write for package.json because there were no changes.') - next() + return } else { - writeFileAtomic(saveTarget, json, next) + return writeFileAtomic(saveTarget, json) } - })) + }) } exports.getSaveType = function (tree, arg) { diff --git a/lib/unarchive.js b/lib/unarchive.js new file mode 100644 index 0000000000000..934ef46485c44 --- /dev/null +++ b/lib/unarchive.js @@ -0,0 +1,34 @@ +'use strict' + +const BB = require('bluebird') + +const MyPrecious = require('libprecious') +const npm = require('./npm.js') +const npmlog = require('npmlog') +const pacoteOpts = require('./config/pacote.js') + +archive.usage = 'npm unarchive' + +archive.completion = (cb) => cb(null, []) + +MyPrecious.PreciousConfig.impl(npm.config, { + get: npm.config.get, + set: npm.config.set, + toPacote (moreOpts) { + return pacoteOpts(moreOpts) + } +}) + +module.exports = archive +function archive (args, cb) { + BB.resolve(_unarchive()).nodeify(cb) +} + +function _unarchive () { + return new MyPrecious({ + config: npm.config, + log: npmlog + }) + .unarchive() + .then(() => console.error('archive information and tarballs removed')) +} diff --git a/node_modules/libprecious/CHANGELOG.md b/node_modules/libprecious/CHANGELOG.md new file mode 100644 index 0000000000000..f95d26ebf09b5 --- /dev/null +++ b/node_modules/libprecious/CHANGELOG.md @@ -0,0 +1,285 @@ +# Change Log + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + + +# [2.0.0](https://github.com/zkat/my-precious/compare/lib-v1.9.0...2.0.0) (2018-07-05) + + +### Bug Fixes + +* **deps:** bump to a fixed pacote version ([024d736](https://github.com/zkat/my-precious/commit/024d736)) + + +### meta + +* drop node4, add node10 ([b73a522](https://github.com/zkat/my-precious/commit/b73a522)) + + +### BREAKING CHANGES + +* this drops support for node@^4 + + + + +# 1.9.0 (2018-03-01) + + +### Bug Fixes + +* **archiveTarball:** generate hashes for existing tarballs anyway ([59932a9](https://github.com/zkat/my-precious/commit/59932a9)) +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** no need to set level here ([d78cda8](https://github.com/zkat/my-precious/commit/d78cda8)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **output:** better output for cli ([2cb5a5a](https://github.com/zkat/my-precious/commit/2cb5a5a)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **tarballs:** fs-safe names for all the spec types ([2f70f11](https://github.com/zkat/my-precious/commit/2f70f11)) +* **tarballs:** only append shortHash for appropriate files ([ab57677](https://github.com/zkat/my-precious/commit/ab57677)) +* **unarchive:** skip manifest lookup for git and directory deps ([7a1e178](https://github.com/zkat/my-precious/commit/7a1e178)) +* **unarchive:** verify integrity when unarchiving, for relevant types only ([2874ee3](https://github.com/zkat/my-precious/commit/2874ee3)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **idempotent:** skip rewriting existing tarballs + remove dangling ones ([a04fb80](https://github.com/zkat/my-precious/commit/a04fb80)) +* **log:** nicer logging and timing info ([4f12053](https://github.com/zkat/my-precious/commit/4f12053)) +* **reset:** add support for resetting to pre-archive state ([7a5953e](https://github.com/zkat/my-precious/commit/7a5953e)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) +* **save:** detect newlines and save them appropriately ([b61ea17](https://github.com/zkat/my-precious/commit/b61ea17)) +* **unarchive:** remove node_modules when unarchiving ([324e8f4](https://github.com/zkat/my-precious/commit/324e8f4)) +* **unarchive:** restore stuff to pre-archive state ([31bc71f](https://github.com/zkat/my-precious/commit/31bc71f)) + + + + +# 1.8.0 (2018-03-01) + + +### Bug Fixes + +* **archiveTarball:** generate hashes for existing tarballs anyway ([59932a9](https://github.com/zkat/my-precious/commit/59932a9)) +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** no need to set level here ([d78cda8](https://github.com/zkat/my-precious/commit/d78cda8)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **output:** better output for cli ([2cb5a5a](https://github.com/zkat/my-precious/commit/2cb5a5a)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **tarballs:** fs-safe names for all the spec types ([2f70f11](https://github.com/zkat/my-precious/commit/2f70f11)) +* **tarballs:** only append shortHash for appropriate files ([ab57677](https://github.com/zkat/my-precious/commit/ab57677)) +* **unarchive:** skip manifest lookup for git and directory deps ([7a1e178](https://github.com/zkat/my-precious/commit/7a1e178)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **idempotent:** skip rewriting existing tarballs + remove dangling ones ([a04fb80](https://github.com/zkat/my-precious/commit/a04fb80)) +* **log:** nicer logging and timing info ([4f12053](https://github.com/zkat/my-precious/commit/4f12053)) +* **reset:** add support for resetting to pre-archive state ([7a5953e](https://github.com/zkat/my-precious/commit/7a5953e)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) +* **save:** detect newlines and save them appropriately ([b61ea17](https://github.com/zkat/my-precious/commit/b61ea17)) +* **unarchive:** restore stuff to pre-archive state ([31bc71f](https://github.com/zkat/my-precious/commit/31bc71f)) + + + + +# 1.7.0 (2018-02-28) + + +### Bug Fixes + +* **archiveTarball:** generate hashes for existing tarballs anyway ([59932a9](https://github.com/zkat/my-precious/commit/59932a9)) +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** no need to set level here ([d78cda8](https://github.com/zkat/my-precious/commit/d78cda8)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **output:** better output for cli ([2cb5a5a](https://github.com/zkat/my-precious/commit/2cb5a5a)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **tarballs:** fs-safe names for all the spec types ([2f70f11](https://github.com/zkat/my-precious/commit/2f70f11)) +* **tarballs:** only append shortHash for appropriate files ([ab57677](https://github.com/zkat/my-precious/commit/ab57677)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **idempotent:** skip rewriting existing tarballs + remove dangling ones ([a04fb80](https://github.com/zkat/my-precious/commit/a04fb80)) +* **log:** nicer logging and timing info ([4f12053](https://github.com/zkat/my-precious/commit/4f12053)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) +* **save:** detect newlines and save them appropriately ([b61ea17](https://github.com/zkat/my-precious/commit/b61ea17)) +* **unarchive:** restore stuff to pre-archive state ([31bc71f](https://github.com/zkat/my-precious/commit/31bc71f)) + + + + +# 1.6.0 (2018-02-24) + + +### Bug Fixes + +* **archiveTarball:** generate hashes for existing tarballs anyway ([59932a9](https://github.com/zkat/my-precious/commit/59932a9)) +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** no need to set level here ([d78cda8](https://github.com/zkat/my-precious/commit/d78cda8)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **output:** better output for cli ([2cb5a5a](https://github.com/zkat/my-precious/commit/2cb5a5a)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **tarballs:** fs-safe names for all the spec types ([2f70f11](https://github.com/zkat/my-precious/commit/2f70f11)) +* **tarballs:** only append shortHash for appropriate files ([ab57677](https://github.com/zkat/my-precious/commit/ab57677)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **idempotent:** skip rewriting existing tarballs + remove dangling ones ([a04fb80](https://github.com/zkat/my-precious/commit/a04fb80)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) +* **save:** detect newlines and save them appropriately ([b61ea17](https://github.com/zkat/my-precious/commit/b61ea17)) + + + + +# 1.5.0 (2018-02-23) + + +### Bug Fixes + +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **idempotent:** skip rewriting existing tarballs + remove dangling ones ([a04fb80](https://github.com/zkat/my-precious/commit/a04fb80)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) + + + + +# 1.4.0 (2018-02-23) + + +### Bug Fixes + +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **integrity:** make it play nicer with npm itself ([ebd2f3d](https://github.com/zkat/my-precious/commit/ebd2f3d)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) + + + + +# 1.3.0 (2018-02-23) + + +### Bug Fixes + +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **tarballInfo:** guard against undefined dep.resolved ([1a4ec12](https://github.com/zkat/my-precious/commit/1a4ec12)) +* **tarballInfo:** mostly idempotent integrity updates ([a3c84a7](https://github.com/zkat/my-precious/commit/a3c84a7)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) + + + + +# 1.2.0 (2018-02-23) + + +### Bug Fixes + +* **config:** default referer -> libprecious ([633f92b](https://github.com/zkat/my-precious/commit/633f92b)) +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **log:** pass log object through ot pacote ([af8f032](https://github.com/zkat/my-precious/commit/af8f032)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) +* **windows:** normalize resolved paths on Windows to use / ([c2da48f](https://github.com/zkat/my-precious/commit/c2da48f)) + + +### Features + +* **archive:** standalone `archiveTarball` + better archive names ([e89546e](https://github.com/zkat/my-precious/commit/e89546e)) +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) +* **save:** add support for --only and --also envs ([ac8cc79](https://github.com/zkat/my-precious/commit/ac8cc79)) + + + + +# 1.1.0 (2018-02-20) + + +### Bug Fixes + +* **deps:** npmlog was missing from cli ([aad8bd9](https://github.com/zkat/my-precious/commit/aad8bd9)) +* **integrity:** generate integrity for all existing algorithms ([987c8b1](https://github.com/zkat/my-precious/commit/987c8b1)) +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) + + +### Features + +* **archive:** write out archives as uncompressed .tar ([e5bfa70](https://github.com/zkat/my-precious/commit/e5bfa70)) + + + + +## 1.0.1 (2018-02-20) + + +### Bug Fixes + +* **saveTar:** got it working ([6f541d6](https://github.com/zkat/my-precious/commit/6f541d6)) diff --git a/node_modules/libprecious/LICENSE.md b/node_modules/libprecious/LICENSE.md new file mode 100644 index 0000000000000..8d28acf866d93 --- /dev/null +++ b/node_modules/libprecious/LICENSE.md @@ -0,0 +1,16 @@ +ISC License + +Copyright (c) npm, Inc. + +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted, provided that the +above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE COPYRIGHT HOLDER DISCLAIMS +ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR +CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE +USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/node_modules/libprecious/README.md b/node_modules/libprecious/README.md new file mode 100644 index 0000000000000..9387f790adb9b --- /dev/null +++ b/node_modules/libprecious/README.md @@ -0,0 +1,45 @@ +[![npm](https://img.shields.io/npm/v/libprecious.svg)](https://npm.im/libprecious) [![license](https://img.shields.io/npm/l/libprecious.svg)](https://npm.im/libprecious) [![Travis](https://img.shields.io/travis/zkat/libprecious.svg)](https://travis-ci.org/zkat/libprecious) [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/zkat/libprecious?svg=true)](https://ci.appveyor.com/project/zkat/libprecious) [![Coverage Status](https://coveralls.io/repos/github/zkat/libprecious/badge.svg?branch=latest)](https://coveralls.io/github/zkat/libprecious?branch=latest) + +[`libprecious`](https://npm.im/libprecious) installs npm projects in a way +that's optimized for continuous integration/deployment/etc scenarios. It gives +up the ability to build its own trees or install packages individually, as well +as other user-oriented features, in exchange for speed, and being more strict +about project state. + +For documentation about the associated command-line tool, see +[`my-precious`](https://npm.im/my-precious). + +## Install + +`$ npm install libprecious` + +## Table of Contents + +* [Features](#features) +* [Contributing](#contributing) +* [API](#api) + +### Features + +* saves all your dependency tarballs to local files +* npm-compatible caching +* errors if `package.json` and `package-lock.json` are out of sync, instead of fixing it like npm does. Essentially provides a `--frozen` install. + +### Contributing + +The libprecious team enthusiastically welcomes contributions and project +participation! There's a bunch of things you can do if you want to contribute! +The [Contributor Guide](CONTRIBUTING.md) has all the information you need for +everything from reporting bugs to contributing entire new features. Please don't +hesitate to jump in if you'd like to, or even ask us questions if something +isn't clear. + +## ACKNOWLEDGEMENTS + +`libprecious` is a reimplementation of the concept of +[`shrinkpack`](https://npm.im/shrinkpack), with the intention of integrating it +directly into npm itself, and using common libraries for increased +compatibility. + +The `libprecious` team is grateful to Jamie Mason and the rest of the +`shrinkpack` developers for their support and encouragement! diff --git a/node_modules/libprecious/index.js b/node_modules/libprecious/index.js new file mode 100644 index 0000000000000..f7dc73e24298f --- /dev/null +++ b/node_modules/libprecious/index.js @@ -0,0 +1,387 @@ +'use strict' + +const BB = require('bluebird') + +const buildLogicalTree = require('npm-logical-tree') +const detectIndent = require('detect-indent') +const detectNewline = require('detect-newline') +const fs = require('graceful-fs') +const getPrefix = require('find-npm-prefix') +const lockVerify = require('lock-verify') +const mkdirp = BB.promisify(require('mkdirp')) +const npa = require('npm-package-arg') +const pacote = require('pacote') +const path = require('path') +const rimraf = BB.promisify(require('rimraf')) +const ssri = require('ssri') +const zlib = require('zlib') + +const readdirAsync = BB.promisify(fs.readdir) +const readFileAsync = BB.promisify(fs.readFile) +const statAsync = BB.promisify(fs.stat) +const writeFileAsync = BB.promisify(fs.writeFile) + +class MyPrecious { + constructor (opts) { + this.opts = opts + this.config = opts.config + + // Stats + this.startTime = Date.now() + this.runTime = 0 + this.timings = {} + this.pkgCount = 0 + this.removed = 0 + + // Misc + this.log = this.opts.log || require('./lib/silentlog.js') + this.pkg = null + this.tree = null + this.archives = new Set() + } + + timedStage (name) { + const start = Date.now() + return BB.resolve(this[name].apply(this, [].slice.call(arguments, 1))) + .tap(() => { + this.timings[name] = Date.now() - start + this.log.info(name, `Done in ${this.timings[name] / 1000}s`) + }) + } + + run () { return this.archive() } + archive () { + return this.timedStage('prepare') + .then(() => this.timedStage('findExisting')) + .then(() => this.timedStage('saveTarballs', this.tree)) + .then(() => this.timedStage('updateLockfile', this.tree)) + .then(() => this.timedStage('cleanupArchives')) + .then(() => this.timedStage('teardown')) + .then(() => { this.runTime = Date.now() - this.startTime }) + .catch(err => { this.timedStage('teardown'); throw err }) + .then(() => this) + } + + unarchive () { + return this.timedStage('prepare') + .then(() => this.timedStage('restoreDependencies', this.tree)) + .then(() => this.timedStage('updateLockfile', this.tree)) + .then(() => this.timedStage('removeTarballs')) + .then(() => this.timedStage('removeModules')) + .then(() => { + this.runTime = Date.now() - this.startTime + this.log.info( + 'run-time', + `total run time: ${this.runTime / 1000}s` + ) + }) + .catch(err => { this.timedStage('teardown'); throw err }) + .then(() => this) + } + + prepare () { + this.log.info('prepare', 'initializing installer') + return ( + this.config.get('prefix') && this.config.get('global') + ? BB.resolve(this.config.get('prefix')) + // There's some Special™ logic around the `--prefix` config when it + // comes from a config file or env vs when it comes from the CLI + : process.argv.some(arg => arg.match(/^\s*--prefix\s*/i)) + ? this.config.get('prefix') + : getPrefix(process.cwd()) + ) + .then(prefix => { + this.prefix = prefix + this.archiveDir = path.join(prefix, 'archived-packages') + this.log.verbose('prepare', 'package prefix: ' + prefix) + return BB.join( + readJson(prefix, 'package.json'), + readJson(prefix, 'package-lock.json', true), + readJson(prefix, 'npm-shrinkwrap.json', true), + (pkg, lock, shrink) => { + pkg._shrinkwrap = shrink || lock + this.pkg = pkg + this.lockName = shrink ? 'npm-shrinkwrap.json' : 'package-lock.json' + } + ) + }) + .then(() => statAsync( + path.join(this.prefix, 'node_modules') + ).catch(err => { if (err.code !== 'ENOENT') { throw err } })) + .then(stat => this.checkLock()) + .then(() => { + this.tree = buildLogicalTree(this.pkg, this.pkg._shrinkwrap) + this.log.silly('tree', this.tree) + }) + } + + teardown () { + return BB.resolve() + } + + checkLock () { + this.log.info('checkLock', 'verifying package-lock data') + const pkg = this.pkg + const prefix = this.prefix + if (!pkg._shrinkwrap || !pkg._shrinkwrap.lockfileVersion) { + return BB.reject( + new Error(`we can only install packages with an existing package-lock.json or npm-shrinkwrap.json with lockfileVersion >= 1. Run an install with npm@5 or later to generate it, then try again.`) + ) + } + return lockVerify(prefix).then(result => { + if (result.status) { + result.warnings.forEach(w => this.log.warn('lockfile', w)) + } else { + throw new Error( + 'we can only install packages when your package.json and package-lock.json or npm-shrinkwrap.json are in sync. Please update your lock file with `npm install` before continuing.\n\n' + + result.warnings.map(w => 'Warning: ' + w).join('\n') + '\n' + + result.errors.join('\n') + '\n' + ) + } + }) + } + + findExisting () { + this.log.info('findExisting', 'checking for existing archived packages') + return readdirAsync(this.archiveDir) + .catch(err => { + if (err.code === 'ENOENT') { return [] } + throw err + }) + .then(existing => { + return BB.all( + existing.filter(f => f.match(/^@/)) + .map(f => { + return readdirAsync(path.join(this.archiveDir, f)) + .then(subfiles => subfiles.map(subf => `${f}/${subf}`)) + }) + ) + .then(scoped => scoped.reduce((acc, scoped) => { + return acc.concat(scoped) + }, existing.filter(f => !f.match(/^@/)))) + }) + .then(allExisting => { this.existingArchives = new Set(allExisting) }) + } + + archiveTarball (spec, dep) { + const pkgPath = this.getTarballPath(dep) + this.log.silly('archiveTarball', `${spec} -> ${pkgPath}`) + const relpath = path.relative(this.archiveDir, pkgPath) + const alreadyExists = this.existingArchives.has( + path.relative(this.archiveDir, pkgPath) + ) + const algorithms = dep.integrity && Object.keys(ssri.parse(dep.integrity)) + this.archives.add(relpath) + return mkdirp(path.dirname(pkgPath)) + .then(() => { + if (alreadyExists) { + this.log.silly('archiveTarball', `archive for ${spec} already exists`) + return ssri.fromStream(fs.createReadStream(pkgPath), {algorithms}) + } + return new BB((resolve, reject) => { + const tardata = pacote.tarball.stream(spec, this.config.toPacote({ + log: this.log, + resolved: dep.resolved && + !dep.resolved.startsWith('file:') && + dep.resolved, + integrity: dep.integrity + })) + const gunzip = zlib.createGunzip() + const sriStream = ssri.integrityStream({algorithms}) + const out = fs.createWriteStream(pkgPath) + let integrity + sriStream.on('integrity', i => { integrity = i }) + tardata.on('error', reject) + gunzip.on('error', reject) + sriStream.on('error', reject) + out.on('error', reject) + out.on('close', () => resolve(integrity)) + tardata + .pipe(gunzip) + .pipe(sriStream) + .pipe(out) + }) + .tap(() => { this.pkgCount++ }) + }) + .then(tarIntegrity => { + const resolvedPath = path.relative(this.prefix, pkgPath) + .replace(/\\/g, '/') + let integrity + if (!dep.integrity) { + integrity = tarIntegrity.toString() + } else if (dep.integrity.indexOf(tarIntegrity.toString()) !== -1) { + // TODO - this is a stopgap until `ssri#concat` (or a new + // `ssri#merge`) become availble. + integrity = dep.integrity + } else { + // concat needs to be in this order 'cause apparently it's what npm + // expects to do. + integrity = tarIntegrity.concat(dep.integrity).toString() + } + return { + resolved: `file:${resolvedPath}`, + integrity + } + }) + } + + restoreDependencies (tree) { + this.log.info('restoreDependencies', 'removing archive details from dependencies locked') + return tree.forEachAsync((dep, next) => { + if (dep.isRoot || dep.bundled) { return next() } + const spec = npa.resolve(dep.name, dep.version, this.prefix) + if (spec.type === 'directory' || spec.type === 'git') { + dep.resolved = null + dep.integrity = null + return next() + } + return pacote.manifest(spec, this.config.toPacote({ + log: this.log, + integrity: ( + spec.type === 'remote' || + spec.registry || + spec.type === 'local' + ) + ? dep.integrity + : null + })) + .then(mani => { + dep.resolved = mani._resolved || null + dep.integrity = mani._integrity || null + }) + .then(next) + }) + } + + getTarballPath (dep) { + let suffix + const spec = npa.resolve(dep.name, dep.version, this.prefix) + if (spec.registry) { + suffix = dep.version + } else if (spec.type === 'remote') { + suffix = 'remote' + } else if (spec.type === 'file') { + suffix = 'file' + } else if (spec.hosted) { + suffix = `${spec.hosted.type}-${spec.hosted.user}-${spec.hosted.project}-${spec.gitCommittish}` + } else if (spec.type === 'git') { + suffix = `git-${spec.gitCommittish}` + } else if (spec.type === 'directory') { + suffix = 'directory' + } + if (dep.integrity && ( + spec.registry || spec.type === 'file' || spec.type === 'remote' + )) { + const split = dep.integrity.split(/\s+/) + const shortHash = ssri.parse(split[split.length - 1], {single: true}) + .hexDigest() + .substr(0, 9) + suffix += `-${shortHash}` + } + const filename = `${dep.name}-${suffix}.tar` + return path.join(this.archiveDir, filename) + } + + saveTarballs (tree) { + this.log.info('saveTarballs', 'archiving packages to', this.archiveDir) + return tree.forEachAsync((dep, next) => { + if (!this.checkDepEnv(dep)) { return } + const spec = npa.resolve(dep.name, dep.version, this.prefix) + if (dep.isRoot || spec.type === 'directory' || dep.bundled) { + return next() + } else { + return this.archiveTarball(spec, dep) + .then(updated => Object.assign(dep, updated)) + .then(() => next()) + } + }, {concurrency: 50, Promise: BB}) + } + + removeTarballs () { + this.log.info('removeTarballs', 'removing tarball archive') + return rimraf(this.archiveDir) + } + + removeModules () { + this.log.info('removeModules', 'removing archive-installed node_modules/') + return rimraf(path.join(this.prefix, 'node_modules')) + } + + checkDepEnv (dep) { + const includeDev = ( + // Covers --dev and --development (from npm config itself) + this.config.get('dev') || + ( + !/^prod(uction)?$/.test(this.config.get('only')) && + !this.config.get('production') + ) || + /^dev(elopment)?$/.test(this.config.get('only')) || + /^dev(elopment)?$/.test(this.config.get('also')) + ) + const includeProd = !/^dev(elopment)?$/.test(this.config.get('only')) + return (dep.dev && includeDev) || (!dep.dev && includeProd) + } + + updateLockfile (tree) { + this.log.info('updateLockfile', 'updating details in lockfile') + tree.forEach((dep, next) => { + if (dep.isRoot) { return next() } + const physDep = dep.address.split(':').reduce((obj, name, i) => { + return obj.dependencies[name] + }, this.pkg._shrinkwrap) + if (dep.resolved) { + physDep.resolved = dep.resolved + } else { + delete physDep.resolved + } + if (dep.integrity) { + physDep.integrity = dep.integrity + } else { + delete physDep.integrity + } + next() + }) + const lockPath = path.join(this.prefix, this.lockName) + return readFileAsync(lockPath, 'utf8') + .then(file => { + const indent = detectIndent(file).indent || 2 + const ending = detectNewline.graceful(file) + return writeFileAsync( + lockPath, + JSON.stringify(this.pkg._shrinkwrap, null, indent) + .replace(/\n/g, ending) + ) + }) + } + + cleanupArchives () { + const removeMe = [] + for (let f of this.existingArchives.values()) { + if (!this.archives.has(f)) { + removeMe.push(f) + } + } + if (removeMe.length) { + this.log.info('cleanupArchives', 'removing', removeMe.length, 'dangling archives') + this.removed = removeMe.length + } + return BB.map(removeMe, f => rimraf(path.join(this.archiveDir, f))) + } +} +module.exports = MyPrecious +module.exports.PreciousConfig = require('./lib/config/npm-config.js').PreciousConfig + +function stripBOM (str) { + return str.replace(/^\uFEFF/, '') +} + +module.exports._readJson = readJson +function readJson (jsonPath, name, ignoreMissing) { + return readFileAsync(path.join(jsonPath, name), 'utf8') + .then(str => JSON.parse(stripBOM(str))) + .catch({code: 'ENOENT'}, err => { + if (!ignoreMissing) { + throw err + } + }) +} diff --git a/node_modules/libprecious/lib/config/npm-config.js b/node_modules/libprecious/lib/config/npm-config.js new file mode 100644 index 0000000000000..0cf0a1503aeb1 --- /dev/null +++ b/node_modules/libprecious/lib/config/npm-config.js @@ -0,0 +1,68 @@ +'use strict' + +const BB = require('bluebird') + +const pacoteOpts = require('./pacote-opts.js') +const protoduck = require('protoduck') +const spawn = require('child_process').spawn + +class NpmConfig extends Map {} + +const PreciousConfig = protoduck.define({ + get: [], + set: [], + toPacote: [] +}, { + name: 'PreciousConfig' +}) +module.exports.PreciousConfig = PreciousConfig + +PreciousConfig.impl(NpmConfig, { + get: Map.prototype.get, + set: Map.prototype.set, + toPacote (opts) { + return pacoteOpts(this, opts) + } +}) + +module.exports.fromObject = fromObj +function fromObj (obj) { + const map = new NpmConfig() + Object.keys(obj).forEach(k => map.set(k, obj[k])) + return map +} + +module.exports.fromNpm = getNpmConfig +function getNpmConfig (argv) { + return new BB((resolve, reject) => { + const npmBin = process.platform === 'win32' ? 'npm.cmd' : 'npm' + const child = spawn(npmBin, [ + 'config', 'ls', '--json', '-l' + // We add argv here to get npm to parse those options for us :D + ].concat(argv || []), { + env: process.env, + cwd: process.cwd(), + stdio: [0, 'pipe', 2] + }) + + let stdout = '' + if (child.stdout) { + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + } + + child.on('error', reject) + child.on('close', (code) => { + if (code === 127) { + reject(new Error('`npm` command not found. Please ensure you have npm@5.4.0 or later installed.')) + } else { + try { + resolve(fromObj(JSON.parse(stdout))) + } catch (e) { + reject(new Error('`npm config ls --json` failed to output json. Please ensure you have npm@5.4.0 or later installed.')) + } + } + }) + }) +} diff --git a/node_modules/libprecious/lib/config/pacote-opts.js b/node_modules/libprecious/lib/config/pacote-opts.js new file mode 100644 index 0000000000000..af55ddce5a4ff --- /dev/null +++ b/node_modules/libprecious/lib/config/pacote-opts.js @@ -0,0 +1,135 @@ +'use strict' + +const Buffer = require('safe-buffer').Buffer + +const crypto = require('crypto') +const path = require('path') + +let effectiveOwner + +const npmSession = crypto.randomBytes(8).toString('hex') + +module.exports = pacoteOpts +function pacoteOpts (npmOpts, moreOpts) { + const ownerStats = calculateOwner() + const opts = { + cache: path.join(npmOpts.get('cache'), '_cacache'), + ca: npmOpts.get('ca'), + cert: npmOpts.get('cert'), + git: npmOpts.get('git'), + key: npmOpts.get('key'), + localAddress: npmOpts.get('local-address'), + loglevel: npmOpts.get('loglevel'), + maxSockets: +(npmOpts.get('maxsockets') || 15), + npmSession: npmSession, + offline: npmOpts.get('offline'), + projectScope: moreOpts.rootPkg && getProjectScope(moreOpts.rootPkg.name), + proxy: npmOpts.get('https-proxy') || npmOpts.get('proxy'), + refer: 'libprecious', + registry: npmOpts.get('registry'), + retry: { + retries: npmOpts.get('fetch-retries'), + factor: npmOpts.get('fetch-retry-factor'), + minTimeout: npmOpts.get('fetch-retry-mintimeout'), + maxTimeout: npmOpts.get('fetch-retry-maxtimeout') + }, + strictSSL: npmOpts.get('strict-ssl'), + userAgent: npmOpts.get('user-agent'), + + dmode: parseInt('0777', 8) & (~npmOpts.get('umask')), + fmode: parseInt('0666', 8) & (~npmOpts.get('umask')), + umask: npmOpts.get('umask') + } + + if (ownerStats.uid != null || ownerStats.gid != null) { + Object.assign(opts, ownerStats) + } + + (npmOpts.forEach ? Array.from(npmOpts.keys()) : npmOpts.keys).forEach(k => { + const authMatchGlobal = k.match( + /^(_authToken|username|_password|password|email|always-auth|_auth)$/ + ) + const authMatchScoped = k[0] === '/' && k.match( + /(.*):(_authToken|username|_password|password|email|always-auth|_auth)$/ + ) + + // if it matches scoped it will also match global + if (authMatchGlobal || authMatchScoped) { + let nerfDart = null + let key = null + let val = null + + if (!opts.auth) { opts.auth = {} } + + if (authMatchScoped) { + nerfDart = authMatchScoped[1] + key = authMatchScoped[2] + val = npmOpts.get(k) + if (!opts.auth[nerfDart]) { + opts.auth[nerfDart] = { + alwaysAuth: !!npmOpts.get('always-auth') + } + } + } else { + key = authMatchGlobal[1] + val = npmOpts.get(k) + opts.auth.alwaysAuth = !!npmOpts.get('always-auth') + } + + const auth = authMatchScoped ? opts.auth[nerfDart] : opts.auth + if (key === '_authToken') { + auth.token = val + } else if (key.match(/password$/i)) { + auth.password = + // the config file stores password auth already-encoded. pacote expects + // the actual username/password pair. + Buffer.from(val, 'base64').toString('utf8') + } else if (key === 'always-auth') { + auth.alwaysAuth = val === 'false' ? false : !!val + } else { + auth[key] = val + } + } + + if (k[0] === '@') { + if (!opts.scopeTargets) { opts.scopeTargets = {} } + opts.scopeTargets[k.replace(/:registry$/, '')] = npmOpts.get(k) + } + }) + + Object.keys(moreOpts || {}).forEach((k) => { + opts[k] = moreOpts[k] + }) + + return opts +} + +function calculateOwner () { + if (!effectiveOwner) { + effectiveOwner = { uid: 0, gid: 0 } + + // Pretty much only on windows + if (!process.getuid) { + return effectiveOwner + } + + effectiveOwner.uid = +process.getuid() + effectiveOwner.gid = +process.getgid() + + if (effectiveOwner.uid === 0) { + if (process.env.SUDO_UID) effectiveOwner.uid = +process.env.SUDO_UID + if (process.env.SUDO_GID) effectiveOwner.gid = +process.env.SUDO_GID + } + } + + return effectiveOwner +} + +function getProjectScope (pkgName) { + const sep = pkgName.indexOf('/') + if (sep === -1) { + return '' + } else { + return pkgName.slice(0, sep) + } +} diff --git a/node_modules/libprecious/lib/silentlog.js b/node_modules/libprecious/lib/silentlog.js new file mode 100644 index 0000000000000..4c9d6c57e8b7d --- /dev/null +++ b/node_modules/libprecious/lib/silentlog.js @@ -0,0 +1,13 @@ +'use strict' + +const noop = Function.prototype +module.exports = { + error: noop, + warn: noop, + info: noop, + verbose: noop, + silly: noop, + http: noop, + pause: noop, + resume: noop +} diff --git a/node_modules/libprecious/package.json b/node_modules/libprecious/package.json new file mode 100644 index 0000000000000..6b17822a7c073 --- /dev/null +++ b/node_modules/libprecious/package.json @@ -0,0 +1,104 @@ +{ + "_from": "libprecious@latest", + "_id": "libprecious@2.0.0", + "_inBundle": false, + "_integrity": "sha512-cAMZBq/oUnP7bOAJCS/EUJcLzVRpJjskpVJawT5Y2H9O1p55VOstd/HmhaAaU/kutFVtv3/i6U2YapnSEMYmxg==", + "_location": "/libprecious", + "_phantomChildren": {}, + "_requested": { + "type": "tag", + "registry": true, + "raw": "libprecious@latest", + "name": "libprecious", + "escapedName": "libprecious", + "rawSpec": "latest", + "saveSpec": null, + "fetchSpec": "latest" + }, + "_requiredBy": [ + "#USER", + "/" + ], + "_resolved": "https://registry.npmjs.org/libprecious/-/libprecious-2.0.0.tgz", + "_shasum": "2bbdf324b4e6861dace215b3e774a23ade2d3949", + "_spec": "libprecious@latest", + "_where": "/Users/zkat/Documents/code/work/npm", + "author": { + "name": "Kat Marchán", + "email": "kzm@sykosomatic.org" + }, + "bugs": { + "url": "https://github.com/zkat/my-precious/issues" + }, + "bundleDependencies": false, + "config": { + "nyc": { + "exclude": [ + "node_modules/**", + "test/**" + ] + } + }, + "dependencies": { + "bluebird": "^3.5.1", + "detect-indent": "^5.0.0", + "detect-newline": "^2.1.0", + "find-npm-prefix": "^1.0.2", + "graceful-fs": "^4.1.11", + "lock-verify": "^2.0.2", + "mkdirp": "^0.5.1", + "npm-logical-tree": "^1.2.1", + "npm-package-arg": "^6.1.0", + "pacote": "^8.1.6", + "protoduck": "^5.0.0", + "rimraf": "^2.6.2", + "ssri": "^6.0.0" + }, + "deprecated": false, + "description": "a local package archive, of my own", + "devDependencies": { + "get-stream": "^3.0.0", + "has-unicode": "^2.0.1", + "mississippi": "^3.0.0", + "nock": "^9.4.1", + "npmlog": "^4.1.2", + "nyc": "^12.0.2", + "require-inject": "^1.4.3", + "standard": "^11.0.1", + "standard-version": "^4.4.0", + "tacks": "^1.2.7", + "tap": "^12.0.1", + "tar-stream": "^1.6.1", + "weallbehave": "^1.2.0", + "weallcontribute": "^1.0.8" + }, + "files": [ + "*.js", + "lib" + ], + "homepage": "https://github.com/zkat/my-precious#readme", + "keywords": [ + "npm", + "package manager", + "caching", + "downloader", + "local registry" + ], + "license": "ISC", + "main": "index.js", + "name": "libprecious", + "repository": { + "type": "git", + "url": "git+https://github.com/zkat/my-precious.git" + }, + "scripts": { + "postrelease": "npm publish && git push --follow-tags", + "prerelease": "npm t", + "pretest": "standard", + "release": "standard-version -s -t 'lib-v'", + "test": "tap -J --nyc-arg=--all --coverage test/*.js", + "update-coc": "weallbehave -o . && git add CODE_OF_CONDUCT.md && git commit -m 'docs(coc): updated CODE_OF_CONDUCT.md'", + "update-contrib": "weallcontribute -o . && git add CONTRIBUTING.md && git commit -m 'docs(contributing): updated CONTRIBUTING.md'" + }, + "version": "2.0.0" +} diff --git a/package-lock.json b/package-lock.json index 9b4218de30b6f..7f5456ef53686 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2414,6 +2414,26 @@ "yargs": "^11.0.0" } }, + "libprecious": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/libprecious/-/libprecious-2.0.0.tgz", + "integrity": "sha512-cAMZBq/oUnP7bOAJCS/EUJcLzVRpJjskpVJawT5Y2H9O1p55VOstd/HmhaAaU/kutFVtv3/i6U2YapnSEMYmxg==", + "requires": { + "bluebird": "^3.5.1", + "detect-indent": "^5.0.0", + "detect-newline": "^2.1.0", + "find-npm-prefix": "^1.0.2", + "graceful-fs": "^4.1.11", + "lock-verify": "^2.0.2", + "mkdirp": "^0.5.1", + "npm-logical-tree": "^1.2.1", + "npm-package-arg": "^6.1.0", + "pacote": "^8.1.6", + "protoduck": "^5.0.0", + "rimraf": "^2.6.2", + "ssri": "^6.0.0" + } + }, "load-json-file": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", diff --git a/package.json b/package.json index 4fcf52652edb1..e491f76f7c3d2 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "libcipm": "^2.0.0", "libnpmhook": "^4.0.1", "libnpx": "^10.2.0", + "libprecious": "^2.0.0", "lock-verify": "^2.0.2", "lockfile": "^1.0.4", "lodash._baseuniq": "~4.6.0", @@ -154,6 +155,7 @@ "bluebird", "bin-links", "chownr", + "cli-columns", "cli-table2", "cmd-shim", "columnify", @@ -185,6 +187,8 @@ "libcipm", "libnpmhook", "libnpx", + "libprecious", + "lock-verify", "lockfile", "lodash._baseindexof", "lodash._baseuniq", @@ -202,6 +206,7 @@ "mkdirp", "mississippi", "move-concurrently", + "node-gyp", "nopt", "normalize-package-data", "npm-audit-report", @@ -235,6 +240,7 @@ "request", "retry", "rimraf", + "safe-buffer", "semver", "sha", "slide", @@ -243,6 +249,7 @@ "ssri", "tar", "text-table", + "tiny-relative-date", "uid-number", "umask", "unique-filename", @@ -255,11 +262,7 @@ "wrappy", "write-file-atomic", "safe-buffer", - "worker-farm", - "tiny-relative-date", - "cli-columns", - "node-gyp", - "lock-verify" + "worker-farm" ], "devDependencies": { "deep-equal": "~1.0.1",