archive: committable offline dependency archive

doc/cli/npm-archive.md

@@ -0,0 +1,85 @@
+npm-archive(1) -- Project-local dependency tarball archive
+===================================
+
+## SYNOPSIS
+
+    npm archive
+
+## EXAMPLE
+
+Make sure you have a package-lock and an up-to-date install:
+
+```
+$ cd ./my/npm/project
+$ npm install
+added 154 packages in 10s
+$ ls | grep package-lock
+```
+
+Run `npm archive` in that project
+
+```
+$ npm archive
+added 1964 packages in 4.103s
+```
+
+Commit the newly-created `archived-packages/` directory and the modified `package-lock.json`
+
+```
+$ git add package-lock.json archived-packages/
+$ git commit -m 'misc: committing dependency archive'
+```
+
+Add a dependency as usual -- its archive will be automatically managed.
+
+```
+$ npm i aubergine
+added 1 package from 1 contributor in 5s
+$ git status
+ M package-lock.json
+ M package.json
+?? archived-packages/aubergine-1.0.1-46c5742af.tar
+$ git add archived-packages package-lock.json package.json
+$ git commit -m 'deps: [email protected]'
+```
+
+The inverse happens when a package is removed.
+
+You can then install normally using `npm-ci(1)` or `npm-install(1)`!
+
+```
+$ npm ci
+added 1965 packages in 10.5s
+```
+
+Finally, you can remove and disable the archive, restoring `package-lock.json` its normal state, by using `npm-unarchive(1)`.
+
+```
+$ npm unarchive
+
+```
+## DESCRIPTION
+
+This command generates a committable archive of your project's dependencies. There are several benefits to this:
+
+1. Offline installs without having to warm up npm's global cache
+2. No need for configuring credentials for dependency fetching
+3. Much faster installs vs standard CI configurations
+4. No need to have a `git` binary present in the system
+5. Reduced download duplication across teams
+
+`npm-archive` works by generating tarballs for your dependencies, unzipping them, and storing them in a directory called `archived-packages/`. It then rewrites your `package-lock.json` (or `npm-shrinkwrap.json`) such that the `resolved` field on those dependencies refers to the path in `archived-packages/`.
+
+npm will detect these `file:` URLs and extract package data directly from them instead of the registry, git repositories, etc.
+
+When installing or removing dependencies, npm will look for `archived-packages/` and switch to an "archive mode", which will automatically update archive files and information on every relevant npm operation. Remember to commit the directory, not just `package-lock.json`!
+
+As an added benefit, `npm-archive` will generate tarballs for all your git dependencies and pre-pack them, meaning npm will not need to invoke the git binary or go through other heavy processes git dependencies go to -- making git deps as fast as registry dependencies when reinstalling from an archive.
+
+If specific tarballs are removed from the archive, npm will fall back to standard behavior for fetching dependencies: first checking its global cache, then going out and fetching the dependency from its origin. To regenerate the tarball for a package after removing it, just reinstall the package while in archive mode.
+
+## SEE ALSO
+
+* npm-unarchive(1)
+* npm-package-locks(5)
+* npm-ci(1)

doc/cli/npm-unarchive.md

@@ -0,0 +1,25 @@
+npm-unarchive(1) -- Restore project to a non-archived state.
+===================================
+
+## SYNOPSIS
+
+    npm unarchive
+
+## EXAMPLE
+
+```
+$ npm unarchive
+archive information and tarballs removed
+```
+## DESCRIPTION
+
+This command undoes the work of `npm-archive(1)` by doing the following:
+
+1. Removes the `archived-packages/` directory.
+2. Restores the entires in `package-lock.json` to use non-`file:` resolved URLs and updates their `integrity` fields.
+3. Removes `node_modules/` to prevent archive-related changes from affecting future installs.
+
+## SEE ALSO
+
+* npm-archive(1)
+* npm-package-locks(5)

doc/misc/npm-config.md

@@ -164,6 +164,18 @@ When "true" submit audit reports alongside `npm install` runs to the default
 registry and all registries configured for scopes.  See the documentation
 for npm-audit(1) for details on what is submitted.
 
+### archive
+
+* Default: true
+* Type: Boolean
+
+If false (with `--no-archive`), an existing `archived-packages/` directory
+will not be modified on save.
+
+This flag has no effect if the archive directory does not alredy exist.
+
+See also npm-archive(1).
+
 ### auth-type
 
 * Default: `'legacy'`

lib/archive.js

@@ -0,0 +1,63 @@
+'use strict'
+
+const BB = require('bluebird')
+
+const MyPrecious = require('libprecious')
+const npm = require('./npm.js')
+const npmlog = require('npmlog')
+const pacoteOpts = require('./config/pacote.js')
+const path = require('path')
+
+const statAsync = BB.promisify(require('fs').stat)
+
+archive.usage = 'npm archive\nnpm archive restore'
+
+archive.completion = (cb) => cb(null, [])
+
+MyPrecious.PreciousConfig.impl(npm.config, {
+  get: npm.config.get,
+  set: npm.config.set,
+  toPacote (moreOpts) {
+    return pacoteOpts(moreOpts)
+  }
+})
+
+module.exports = archive
+function archive (args, cb) {
+  BB.resolve(_archive()).nodeify(cb)
+}
+
+function _archive (args) {
+  // TODO - is this the right path?...
+  return statAsync(path.join(npm.prefix, 'archived-packages'))
+  .catch((err) => { if (err.code !== 'ENOENT') { throw err } })
+  .then((stat) => {
+    const archiveExists = stat && stat.isDirectory()
+    return new MyPrecious({
+      config: npm.config,
+      log: npmlog
+    })
+    .run()
+    .then((details) => {
+      if (!archiveExists) {
+        npmlog.notice('archive', 'created new package archive as `archived-packages/`. Future installations will prioritize packages in this directory.')
+      }
+      const clauses = []
+      if (!details.pkgCount && !details.removed) {
+        clauses.push('done')
+      }
+      if (details.pkgCount) {
+        clauses.push(`archived ${details.pkgCount} package${
+          details.pkgCount === 1 ? '' : 's'
+        }`)
+      }
+      if (details.removed) {
+        clauses.push(`cleaned up ${details.pkgCount} archive${
+          details.removed === 1 ? '' : 's'
+        }`)
+      }
+      const time = details.runTime / 1000
+      console.error(`${clauses.join(' and ')} in ${time}s`)
+    })
+  })
+}

lib/config/cmd-list.js

@@ -21,6 +21,7 @@ var shorthands = {
 }
 
 var affordances = {
+  'arr': 'archive',
   'la': 'ls',
   'll': 'ls',
   'verison': 'version',
@@ -52,6 +53,8 @@ var affordances = {
 
 // these are filenames in .
 var cmdList = [
+  'archive',
+  'unarchive',
   'ci',
   'install',
   'install-test',

lib/config/defaults.js

@@ -110,6 +110,7 @@ Object.defineProperty(exports, 'defaults', {get: function () {
     'always-auth': false,
     also: null,
     audit: true,
+    archive: true,
     'auth-type': 'legacy',
 
     'bin-links': true,
@@ -255,6 +256,7 @@ exports.types = {
   'always-auth': Boolean,
   also: [null, 'dev', 'development'],
   audit: Boolean,
+  archive: Boolean,
   'auth-type': ['legacy', 'sso', 'saml', 'oauth'],
   'bin-links': Boolean,
   browser: [null, String],

lib/install.js

@@ -645,9 +645,9 @@ Installer.prototype.saveToDependencies = function (cb) {
   if (this.failing) return cb()
   log.silly('install', 'saveToDependencies')
   if (this.saveOnlyLock) {
-    saveShrinkwrap(this.idealTree, cb)
+    return saveShrinkwrap(this.idealTree).nodeify(cb)
   } else {
-    saveRequested(this.idealTree, cb)
+    return saveRequested(this.idealTree).nodeify(cb)
   }
 }