From 5839c36d3f9fd737adbfd7c6e54b2c02840b2a41 Mon Sep 17 00:00:00 2001 From: Jeremy Judeaux Date: Wed, 21 Jun 2017 05:02:38 +0800 Subject: [PATCH] Run install and pack when fetching git dependencies with a prepare script. (#3553) --- __tests__/fetchers.js | 26 ++++++++ src/cli/commands/install.js | 16 +++-- src/cli/commands/pack.js | 12 +++- src/config.js | 5 ++ src/fetchers/git-fetcher.js | 130 +++++++++++++++++++++++++++++++++--- 5 files changed, 172 insertions(+), 17 deletions(-) diff --git a/__tests__/fetchers.js b/__tests__/fetchers.js index bf9489b95b..8c1c36e31c 100644 --- a/__tests__/fetchers.js +++ b/__tests__/fetchers.js @@ -76,6 +76,32 @@ test('GitFetcher.fetch', async () => { expect(name).toBe('beeper'); }); +test('GitFetcher.fetch with prepare script', async () => { + const dir = await mkdir('git-fetcher-with-prepare'); + const fetcher = new GitFetcher( + dir, + { + type: 'git', + reference: 'https://github.com/Volune/test-js-git-repo', + hash: '96e838bcc908ed424666b4b04efe802fd4c8bccd', + registry: 'npm', + }, + (await Config.create()), + ); + await fetcher.fetch(); + const name = (await fs.readJson(path.join(dir, 'package.json'))).name; + expect(name).toBe('test-js-git-repo'); + const dependencyName = (await fs.readJson(path.join(dir, 'dependency-package.json'))).name; + expect(dependencyName).toBe('beeper'); + // The file "prepare.js" is not in "files" list + expect(await fs.exists(path.join(dir, 'prepare.js'))).toBe(false); + // Check executed lifecycle scripts + expect(await fs.exists(path.join(dir, 'generated', 'preinstall'))).toBe(true); + expect(await fs.exists(path.join(dir, 'generated', 'install'))).toBe(true); + expect(await fs.exists(path.join(dir, 'generated', 'postinstall'))).toBe(true); + expect(await fs.exists(path.join(dir, 'generated', 'prepublish'))).toBe(false); +}); + test('TarballFetcher.fetch', async () => { const dir = await mkdir('tarball-fetcher'); const fetcher = new TarballFetcher( diff --git a/src/cli/commands/install.js b/src/cli/commands/install.js index e61db5838b..fa64a3965e 100644 --- a/src/cli/commands/install.js +++ b/src/cli/commands/install.js @@ -823,6 +823,13 @@ export function setFlags(commander: Object) { commander.option('-T, --save-tilde', 'DEPRECATED'); } +export async function install(config: Config, reporter: Reporter, flags: Object, lockfile: Lockfile): Promise { + await wrapLifecycle(config, flags, async () => { + const install = new Install(flags, config, reporter, lockfile); + await install.init(); + }); +} + export async function run(config: Config, reporter: Reporter, flags: Object, args: Array): Promise { let lockfile; if (flags.lockfile === false) { @@ -855,10 +862,7 @@ export async function run(config: Config, reporter: Reporter, flags: Object, arg throw new MessageError(reporter.lang('installCommandRenamed', `yarn ${command} ${exampleArgs.join(' ')}`)); } - await wrapLifecycle(config, flags, async () => { - const install = new Install(flags, config, reporter, lockfile); - await install.init(); - }); + await install(config, reporter, flags, lockfile); } export async function wrapLifecycle(config: Config, flags: Object, factory: () => Promise): Promise { @@ -871,7 +875,9 @@ export async function wrapLifecycle(config: Config, flags: Object, factory: () = await config.executeLifecycleScript('postinstall'); if (!config.production) { - await config.executeLifecycleScript('prepublish'); + if (!config.disablePrepublish) { + await config.executeLifecycleScript('prepublish'); + } await config.executeLifecycleScript('prepare'); } } diff --git a/src/cli/commands/pack.js b/src/cli/commands/pack.js index 2a90afa059..537a0c44ea 100644 --- a/src/cli/commands/pack.js +++ b/src/cli/commands/pack.js @@ -48,7 +48,10 @@ const NEVER_IGNORE = ignoreLinesToRegex([ '!/+(changes|changelog|history)*', ]); -export async function pack(config: Config, dir: string): Promise { +export async function packTarball( + config: Config, + {mapHeader}: {mapHeader?: Object => Object} = {}, +): Promise { const pkg = await config.readRootManifest(); const {bundledDependencies, main, files: onlyFiles} = pkg; @@ -123,10 +126,15 @@ export async function pack(config: Config, dir: string): Promise header.name = `package${suffix}`; delete header.uid; delete header.gid; - return header; + return mapHeader ? mapHeader(header) : header; }, }); + return packer; +} + +export async function pack(config: Config, dir: string): Promise { + const packer = await packTarball(config); const compressor = packer.pipe(new zlib.Gzip()); return compressor; diff --git a/src/config.js b/src/config.js index bf2e752ce3..d57c6b593d 100644 --- a/src/config.js +++ b/src/config.js @@ -39,6 +39,7 @@ export type ConfigOptions = { ignoreEngines?: boolean, cafile?: ?string, production?: boolean, + disablePrepublish?: boolean, binLinks?: boolean, networkConcurrency?: number, childConcurrency?: number, @@ -144,6 +145,8 @@ export default class Config { production: boolean; + disablePrepublish: boolean; + nonInteractive: boolean; workspacesEnabled: boolean; @@ -328,6 +331,8 @@ export default class Config { this.ignorePlatform = !!opts.ignorePlatform; this.ignoreScripts = !!opts.ignoreScripts; + this.disablePrepublish = !!opts.disablePrepublish; + this.nonInteractive = !!opts.nonInteractive; this.requestManager.setOptions({ diff --git a/src/fetchers/git-fetcher.js b/src/fetchers/git-fetcher.js index 02642c1bda..f2e675aab7 100644 --- a/src/fetchers/git-fetcher.js +++ b/src/fetchers/git-fetcher.js @@ -7,6 +7,10 @@ import Git from '../util/git.js'; import * as fsUtil from '../util/fs.js'; import * as constants from '../constants.js'; import * as crypto from '../util/crypto.js'; +import {install} from '../cli/commands/install.js'; +import Lockfile from '../lockfile/wrapper.js'; +import Config from '../config.js'; +import {packTarball} from '../cli/commands/pack.js'; const tarFs = require('tar-fs'); const url = require('url'); @@ -15,6 +19,8 @@ const fs = require('fs'); const invariant = require('invariant'); +const PACKED_FLAG = '1'; + export default class GitFetcher extends BaseFetcher { async setupMirrorFromCache(): Promise { const tarballMirrorPath = this.getTarballMirrorPath(); @@ -90,11 +96,7 @@ export default class GitFetcher extends BaseFetcher { } return new Promise((resolve, reject) => { - const untarStream = tarFs.extract(this.dest, { - dmode: 0o555, // all dirs should be readable - fmode: 0o444, // all files should be readable - chown: false, // don't chown. just leave as it is - }); + const untarStream = this._createUntarStream(this.dest); const hashStream = new crypto.HashStream(); @@ -131,11 +133,123 @@ export default class GitFetcher extends BaseFetcher { const gitUrl = Git.npmUrlToGitUrl(this.reference); const git = new Git(this.config, gitUrl, hash); await git.init(); - await git.clone(this.dest); + + const manifestFile = await git.getFile('package.json'); + if (!manifestFile) { + throw new MessageError(this.reporter.lang('couldntFindPackagejson', gitUrl)); + } + const scripts = JSON.parse(manifestFile).scripts; + const hasPrepareScript = Boolean(scripts && scripts.prepare); + + if (hasPrepareScript) { + await this.fetchFromInstallAndPack(git); + } else { + await this.fetchFromGitArchive(git); + } + + return { + hash, + }; + } + + async fetchFromInstallAndPack(git: Git): Promise { + const prepareDirectory = this.config.getTemp(`${crypto.hash(git.gitUrl.repository)}.${git.hash}.prepare`); + await fsUtil.unlink(prepareDirectory); + + await git.clone(prepareDirectory); + + const [prepareConfig, prepareLockFile] = await Promise.all([ + Config.create( + { + cwd: prepareDirectory, + disablePrepublish: true, + }, + this.reporter, + ), + Lockfile.fromDirectory(prepareDirectory, this.reporter), + ]); + await install(prepareConfig, this.reporter, {}, prepareLockFile); const tarballMirrorPath = this.getTarballMirrorPath(); const tarballCachePath = this.getTarballCachePath(); + if (tarballMirrorPath) { + await this._packToTarball(prepareConfig, tarballMirrorPath); + } + if (tarballCachePath) { + await this._packToTarball(prepareConfig, tarballCachePath); + } + + await this._packToDirectory(prepareConfig, this.dest); + + await fsUtil.unlink(prepareDirectory); + } + + async _packToTarball(config: Config, path: string): Promise { + const tarballStream = await this._createTarballStream(config); + await new Promise((resolve, reject) => { + const writeStream = fs.createWriteStream(path); + tarballStream.on('error', reject); + writeStream.on('error', reject); + writeStream.on('end', resolve); + writeStream.on('open', () => { + tarballStream.pipe(writeStream); + }); + writeStream.once('finish', resolve); + }); + } + + async _packToDirectory(config: Config, dest: string): Promise { + const tarballStream = await this._createTarballStream(config); + await new Promise((resolve, reject) => { + const untarStream = this._createUntarStream(dest); + tarballStream.on('error', reject); + untarStream.on('error', reject); + untarStream.on('end', resolve); + untarStream.once('finish', resolve); + tarballStream.pipe(untarStream); + }); + } + + _createTarballStream(config: Config): Promise { + let savedPackedHeader = false; + return packTarball(config, { + mapHeader(header: Object): Object { + if (!savedPackedHeader) { + savedPackedHeader = true; + header.pax = header.pax || {}; + // add a custom data on the first header + // in order to distinguish a tar from "git archive" and a tar from "pack" command + header.pax.packed = PACKED_FLAG; + } + return header; + }, + }); + } + + _createUntarStream(dest: string): stream$Writable { + const PREFIX = 'package/'; + let isPackedTarball = undefined; + return tarFs.extract(dest, { + dmode: 0o555, // all dirs should be readable + fmode: 0o444, // all files should be readable + chown: false, // don't chown. just leave as it is + map: header => { + if (isPackedTarball === undefined) { + isPackedTarball = header.pax && header.pax.packed === PACKED_FLAG; + } + if (isPackedTarball) { + header.name = header.name.substr(PREFIX.length); + } + }, + }); + } + + async fetchFromGitArchive(git: Git): Promise { + await git.clone(this.dest); + const tarballMirrorPath = this.getTarballMirrorPath(); + const tarballCachePath = this.getTarballCachePath(); + if (tarballMirrorPath) { await git.archive(tarballMirrorPath); } @@ -143,10 +257,6 @@ export default class GitFetcher extends BaseFetcher { if (tarballCachePath) { await git.archive(tarballCachePath); } - - return { - hash, - }; } async _fetch(): Promise {