diff --git a/__tests__/commands/install/integration-deduping.js b/__tests__/commands/install/integration-deduping.js index 8e099e0a37..8f6e78b6f2 100644 --- a/__tests__/commands/install/integration-deduping.js +++ b/__tests__/commands/install/integration-deduping.js @@ -221,7 +221,7 @@ test.concurrent('install should not hardlink repeated dependencies if linkDuplic }); }); -test.concurrent('install should not copy node_modules when hardlinking', (): Promise => { +test.concurrent('install should not crash when hardlinking deep structures', (): Promise => { // https://github.com/yarnpkg/yarn/issues/2734 // A@1 -> B@1 -> C@1 // -> C@2 @@ -239,13 +239,13 @@ test.concurrent('install should not copy node_modules when hardlinking', (): Pro }); }); -test.concurrent('install should not hardlink full package structure', (): Promise => { +test.concurrent('install should consider different hoisting with --link-duplicate', (): Promise => { // https://github.com/yarnpkg/yarn/issues/2734 // A@1 -> B@1 -> C@1 // -> C@2 // B@2 // C@3 - // D@1 -> B@1 (hardlink) + // D@1 -> B@1 (hardlink) -> *C@1* (redundant) // -> C@1 (hardlink) return runInstall({linkDuplicates: true}, 'hardlink-collision-2', async config => { let a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/package.json')); @@ -254,6 +254,46 @@ test.concurrent('install should not hardlink full package structure', (): Promis a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/node_modules/c/package.json')); d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/c/package.json')); expect(a_1.ino).toEqual(d_1.ino); - expect(await fs.exists(path.join(config.cwd, 'node_modules/d/node_modules/b/node_modules/c/package.json'))).toBe(false); + // this is redundant but we are ok with it + expect(await fs.exists(path.join(config.cwd, 'node_modules/d/node_modules/b/node_modules/c/package.json'))).toBe(true); + }); +}); + +test.concurrent('install should consider different hoisting with --link-duplicate 2', (): Promise => { + // https://github.com/yarnpkg/yarn/issues/2734 + // A@1 -> B@1 + // -> C@1 + // B@2 + // C@3 + // D@1 -> B@1 (hardlink) -> C@1 (hardlink) + // -> C@2 + return runInstall({linkDuplicates: true}, 'hardlink-collision-3', async config => { + let a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/package.json')); + let d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/b/package.json')); + expect(a_1.ino).toEqual(d_1.ino); + a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/c/package.json')); + d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/b/node_modules/c/package.json')); + expect(a_1.ino).toEqual(d_1.ino); + }); +}); + +test.concurrent('install should not hardlink full package structure', (): Promise => { + // https://github.com/yarnpkg/yarn/issues/2734 + // A@1 -> B@1 -> C@1 -> (bundle leftpad) + // -> C@2 + // B@2 + // C@3 + // D@1 -> B@1 (hardlink) -> C@1 (hardlink) -> (bundle leftpad) (hardlink) + // -> C@2 + return runInstall({linkDuplicates: true}, 'hardlink-collision-with-bundled', async config => { + let a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/package.json')); + let d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/b/package.json')); + expect(a_1.ino).toEqual(d_1.ino); + a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/node_modules/c/package.json')); + d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/b/node_modules/c/package.json')); + expect(a_1.ino).toEqual(d_1.ino); + a_1 = await fs.stat(path.join(config.cwd, 'node_modules/a/node_modules/b/node_modules/c/node_modules/left-pad/package.json')); + d_1 = await fs.stat(path.join(config.cwd, 'node_modules/d/node_modules/b/node_modules/c/node_modules/left-pad/package.json')); + expect(a_1.ino).toEqual(d_1.ino); }); }); diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/a-1/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/a-1/package.json new file mode 100644 index 0000000000..f724d1e272 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/a-1/package.json @@ -0,0 +1,7 @@ +{ + "name": "a", + "version": "1.0.0", + "dependencies": { + "b": "file:../b-1" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/b-1/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/b-1/package.json new file mode 100644 index 0000000000..ad3074b214 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/b-1/package.json @@ -0,0 +1,7 @@ +{ + "name": "b", + "version": "1.0.0", + "dependencies": { + "c": "file:../c-1" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/b-2/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/b-2/package.json new file mode 100644 index 0000000000..57549744f3 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/b-2/package.json @@ -0,0 +1,4 @@ +{ + "name": "b", + "version": "2.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/c-1/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/c-1/package.json new file mode 100644 index 0000000000..abd3384930 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/c-1/package.json @@ -0,0 +1,4 @@ +{ + "name": "c", + "version": "1.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/c-2/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/c-2/package.json new file mode 100644 index 0000000000..eede1fcd97 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/c-2/package.json @@ -0,0 +1,4 @@ +{ + "name": "c", + "version": "2.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/c-3/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/c-3/package.json new file mode 100644 index 0000000000..6915ecb021 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/c-3/package.json @@ -0,0 +1,4 @@ +{ + "name": "c", + "version": "3.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/deps/d-1/package.json b/__tests__/fixtures/install/hardlink-collision-3/deps/d-1/package.json new file mode 100644 index 0000000000..eae1f73732 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/deps/d-1/package.json @@ -0,0 +1,8 @@ +{ + "name": "a", + "version": "2.0.0", + "dependencies": { + "b": "file:../b-1", + "c": "file:../c-2" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-3/package.json b/__tests__/fixtures/install/hardlink-collision-3/package.json new file mode 100644 index 0000000000..9b49885a10 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-3/package.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "a": "file:deps/a-1", + "b": "file:deps/b-2", + "c": "file:deps/c-3", + "d": "file:deps/d-1" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/a-1/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/a-1/package.json new file mode 100644 index 0000000000..c2c8651d69 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/a-1/package.json @@ -0,0 +1,8 @@ +{ + "name": "a", + "version": "1.0.0", + "dependencies": { + "b": "file:../b-1", + "c": "file:../c-2" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-1/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-1/package.json new file mode 100644 index 0000000000..f9f1b907dc --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-1/package.json @@ -0,0 +1,7 @@ +{ + "name": "b", + "version": "1.0.0", + "dependencies": { + "c": "file:../c-1/c-1.0.0.tgz" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-2/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-2/package.json new file mode 100644 index 0000000000..57549744f3 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/b-2/package.json @@ -0,0 +1,4 @@ +{ + "name": "b", + "version": "2.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-1/c-1.0.0.tgz b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-1/c-1.0.0.tgz new file mode 100644 index 0000000000..158e620d32 Binary files /dev/null and b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-1/c-1.0.0.tgz differ diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-2/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-2/package.json new file mode 100644 index 0000000000..eede1fcd97 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-2/package.json @@ -0,0 +1,4 @@ +{ + "name": "c", + "version": "2.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-3/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-3/package.json new file mode 100644 index 0000000000..6915ecb021 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/c-3/package.json @@ -0,0 +1,4 @@ +{ + "name": "c", + "version": "3.0.0" +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/d-1/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/d-1/package.json new file mode 100644 index 0000000000..eae1f73732 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/deps/d-1/package.json @@ -0,0 +1,8 @@ +{ + "name": "a", + "version": "2.0.0", + "dependencies": { + "b": "file:../b-1", + "c": "file:../c-2" + } +} diff --git a/__tests__/fixtures/install/hardlink-collision-with-bundled/package.json b/__tests__/fixtures/install/hardlink-collision-with-bundled/package.json new file mode 100644 index 0000000000..9b49885a10 --- /dev/null +++ b/__tests__/fixtures/install/hardlink-collision-with-bundled/package.json @@ -0,0 +1,8 @@ +{ + "dependencies": { + "a": "file:deps/a-1", + "b": "file:deps/b-2", + "c": "file:deps/c-3", + "d": "file:deps/d-1" + } +} diff --git a/src/fetchers/tarball-fetcher.js b/src/fetchers/tarball-fetcher.js index 51d53029b5..cb1dfdf321 100644 --- a/src/fetchers/tarball-fetcher.js +++ b/src/fetchers/tarball-fetcher.js @@ -14,6 +14,7 @@ const url = require('url'); const fs = require('fs'); const stream = require('stream'); const gunzip = require('gunzip-maybe'); +import {removePrefix} from '../util/misc.js'; export default class TarballFetcher extends BaseFetcher { async setupMirrorFromCache(): Promise { @@ -174,13 +175,16 @@ export default class TarballFetcher extends BaseFetcher { } async _fetch(): Promise { + const isFilePath = this.reference.startsWith('file:'); + this.reference = removePrefix(this.reference, 'file:'); const urlParse = url.parse(this.reference); - const isFilePath = urlParse.protocol + // legacy support for local paths in yarn.lock entries + const isRelativePath = urlParse.protocol ? urlParse.protocol.match(/^[a-z]:$/i) : urlParse.pathname ? urlParse.pathname.match(/^(?:\.{1,2})?[\\\/]/) : false; - if (isFilePath) { + if (isFilePath || isRelativePath) { return this.fetchFromLocal(this.reference); } diff --git a/src/resolvers/exotics/tarball-resolver.js b/src/resolvers/exotics/tarball-resolver.js index b3eb45adeb..12b40f0c18 100644 --- a/src/resolvers/exotics/tarball-resolver.js +++ b/src/resolvers/exotics/tarball-resolver.js @@ -5,7 +5,6 @@ import type PackageRequest from '../../package-request.js'; import TarballFetcher from '../../fetchers/tarball-fetcher.js'; import ExoticResolver from './exotic-resolver.js'; import Git from './git-resolver.js'; -import {removePrefix} from '../../util/misc.js'; import guessName from '../../util/guess-name.js'; import * as versionUtil from '../../util/version.js'; import * as crypto from '../../util/crypto.js'; @@ -52,7 +51,7 @@ export default class TarballResolver extends ExoticResolver { return shrunk; } - const url = removePrefix(this.url, 'file:'); + const url = this.url; let {hash, registry} = this; let pkgJson; diff --git a/src/util/fs.js b/src/util/fs.js index f1123ec872..386ba6c268 100644 --- a/src/util/fs.js +++ b/src/util/fs.js @@ -360,6 +360,11 @@ async function buildActionsForHardlink( const {src, dest} = data; const onFresh = data.onFresh || noop; const onDone = data.onDone || noop; + if (files.has(dest)) { + // don't hardlink a file twice + onDone(); + return; + } files.add(dest); if (events.ignoreBasenames.indexOf(path.basename(src)) >= 0) { @@ -450,7 +455,6 @@ async function buildActionsForHardlink( // push all files to queue invariant(srcFiles, 'src files not initialised'); - srcFiles = srcFiles.filter(f => f !== 'node_modules'); let remaining = srcFiles.length; if (!remaining) { onDone();