From dbf0e64ca280a477b05ffc7e4f259fb94fd4fbd2 Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Tue, 27 Aug 2024 11:04:03 -0400 Subject: [PATCH 01/12] fix(core): filter branch in preparation for nx import --- e2e/nx/src/import.test.ts | 52 ++++++- .../src/command-line/import/command-object.ts | 5 + packages/nx/src/command-line/import/import.ts | 6 + .../import/utils/prepare-source-repo.ts | 130 +++++++----------- packages/nx/src/utils/git-utils.ts | 60 ++++---- packages/nx/src/utils/squash.ts | 14 -- 6 files changed, 135 insertions(+), 132 deletions(-) delete mode 100644 packages/nx/src/utils/squash.ts diff --git a/e2e/nx/src/import.test.ts b/e2e/nx/src/import.test.ts index 2bb5196d589d1..54ad7ea3874e3 100644 --- a/e2e/nx/src/import.test.ts +++ b/e2e/nx/src/import.test.ts @@ -8,7 +8,7 @@ import { updateFile, e2eCwd, } from '@nx/e2e/utils'; -import { mkdirSync, rmdirSync } from 'fs'; +import { writeFileSync, mkdirSync, rmdirSync } from 'fs'; import { execSync } from 'node:child_process'; import { join } from 'path'; @@ -86,4 +86,54 @@ describe('Nx Import', () => { ); runCLI(`vite:build created-vite-app`); }); + + it('should be able to import two directories from same repo', () => { + // Setup repo with two packages: a and b + const repoPath = join(tempImportE2ERoot, 'repo'); + mkdirSync(repoPath, { recursive: true }); + writeFileSync(join(repoPath, 'README.md'), `# Repo`); + execSync(`git init`, { + cwd: repoPath, + }); + execSync(`git add .`, { + cwd: repoPath, + }); + execSync(`git commit -am "initial commit"`, { + cwd: repoPath, + }); + execSync(`git checkout -b main`, { + cwd: repoPath, + }); + mkdirSync(join(repoPath, 'packages/a'), { recursive: true }); + writeFileSync(join(repoPath, 'packages/a/README.md'), `# A`); + execSync(`git add packages/a`, { + cwd: repoPath, + }); + execSync(`git commit -m "add package a"`, { + cwd: repoPath, + }); + mkdirSync(join(repoPath, 'packages/b'), { recursive: true }); + writeFileSync(join(repoPath, 'packages/b/README.md'), `# B`); + execSync(`git add packages/b`, { + cwd: repoPath, + }); + execSync(`git commit -m "add package b"`, { + cwd: repoPath, + }); + + runCLI( + `import ${repoPath} packages/a --ref main --source packages/a --no-interactive`, + { + verbose: true, + } + ); + runCLI( + `import ${repoPath} packages/b --ref main --source packages/b --no-interactive`, + { + verbose: true, + } + ); + + checkFilesExist('packages/a/README.md', 'packages/b/README.md'); + }); }); diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts index 0158249d09102..0914ecbdd1c1d 100644 --- a/packages/nx/src/command-line/import/command-object.ts +++ b/packages/nx/src/command-line/import/command-object.ts @@ -28,6 +28,11 @@ export const yargsImportCommand: CommandModule = { type: 'string', description: 'The branch from the source repository to import', }) + .option('depth', { + type: 'number', + description: + 'The depth to clone the source repository (limit this for faster git clone)', + }) .option('interactive', { type: 'boolean', description: 'Interactive mode', diff --git a/packages/nx/src/command-line/import/import.ts b/packages/nx/src/command-line/import/import.ts index ee120e2d88487..2bcc4cbd29841 100644 --- a/packages/nx/src/command-line/import/import.ts +++ b/packages/nx/src/command-line/import/import.ts @@ -41,6 +41,10 @@ export interface ImportOptions { * The directory in the destination repo to import into */ destination: string; + /** + * The depth to clone the source repository (limit this for faster clone times) + */ + depth: number; verbose: boolean; interactive: boolean; @@ -101,6 +105,7 @@ export async function importHandler(options: ImportOptions) { try { sourceGitClient = await cloneFromUpstream(sourceRemoteUrl, sourceRepoPath, { originName: importRemoteName, + depth: options.depth, }); } catch (e) { spinner.fail(`Failed to clone ${sourceRemoteUrl} into ${sourceRepoPath}`); @@ -149,6 +154,7 @@ export async function importHandler(options: ImportOptions) { name: 'destination', message: 'Where in this workspace should the code be imported into?', required: true, + initial: source ? source : undefined, }, ]) ).destination; diff --git a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts index 25cae28176531..f416b77abe13d 100644 --- a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts +++ b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts @@ -25,108 +25,72 @@ export async function prepareSourceRepo( new: true, base: `${originName}/${ref}`, }); + spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`); const relativeSourceDir = relative( gitClient.root, join(gitClient.root, source) ); + if (relativeSourceDir !== '') { + spinner.start( + `Filtering git history to only include files in ${relativeSourceDir}` + ); + await gitClient.filterBranch(relativeSourceDir, tempImportBranch); + spinner.succeed( + `Filtered git history to only include files in ${relativeSourceDir}` + ); + } + const destinationInSource = join(gitClient.root, relativeDestination); spinner.start(`Moving files and git history to ${destinationInSource}`); - if (relativeSourceDir === '') { - const files = await gitClient.getGitFiles('.'); - try { - await rm(destinationInSource, { - recursive: true, - }); - } catch {} - await mkdir(destinationInSource, { recursive: true }); - const gitignores = new Set(); - for (const file of files) { - if (basename(file) === '.gitignore') { - gitignores.add(file); - continue; - } - spinner.start( - `Moving files and git history to ${destinationInSource}: ${file}` - ); - - const newPath = join(destinationInSource, file); - await mkdir(dirname(newPath), { recursive: true }); - try { - await gitClient.move(file, newPath); - } catch { - await wait(100); - await gitClient.move(file, newPath); - } + // The result of filter-branch will contain only the files in the subdirectory at its root. + const files = await gitClient.getGitFiles('.'); + try { + await rm(destinationInSource, { + recursive: true, + }); + } catch {} + await mkdir(destinationInSource, { recursive: true }); + const gitignores = new Set(); + for (const file of files) { + if (basename(file) === '.gitignore') { + gitignores.add(file); + continue; } - - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` + spinner.start( + `Moving files and git history to ${destinationInSource}: ${file}` ); - for (const gitignore of gitignores) { - await gitClient.move(gitignore, join(destinationInSource, gitignore)); - } - await gitClient.amendCommit(); - for (const gitignore of gitignores) { - await copyFile( - join(destinationInSource, gitignore), - join(gitClient.root, gitignore) - ); - } - } else { - let needsSquash = false; - const needsMove = destinationInSource !== join(gitClient.root, source); - if (needsMove) { - try { - await rm(destinationInSource, { - recursive: true, - }); - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` - ); - needsSquash = true; - } catch {} + const newPath = join(destinationInSource, file); - await mkdir(destinationInSource, { recursive: true }); + await mkdir(dirname(newPath), { recursive: true }); + try { + await gitClient.move(file, newPath); + } catch { + await wait(100); + await gitClient.move(file, newPath); } + } - const files = await gitClient.getGitFiles('.'); - for (const file of files) { - if (file === '.gitignore') { - continue; - } - spinner.start( - `Moving files and git history to ${destinationInSource}: ${file}` - ); + await gitClient.commit( + `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` + ); - if (!relative(source, file).startsWith('..')) { - if (needsMove) { - const newPath = join(destinationInSource, file); + for (const gitignore of gitignores) { + await gitClient.move(gitignore, join(destinationInSource, gitignore)); + } - await mkdir(dirname(newPath), { recursive: true }); - try { - await gitClient.move(file, newPath); - } catch { - await wait(100); - await gitClient.move(file, newPath); - } - } - } else { - await rm(join(gitClient.root, file), { - recursive: true, - }); - } - } - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` + await gitClient.amendCommit(); + + for (const gitignore of gitignores) { + await copyFile( + join(destinationInSource, gitignore), + join(gitClient.root, gitignore) ); - if (needsSquash) { - await gitClient.squashLastTwoCommits(); - } } + spinner.succeed( `${sourceRemoteUrl} has been prepared to be imported into this workspace on a temporary branch: ${tempImportBranch} in ${gitClient.root}` ); diff --git a/packages/nx/src/utils/git-utils.ts b/packages/nx/src/utils/git-utils.ts index f4811781f4aee..1d58d3e90570d 100644 --- a/packages/nx/src/utils/git-utils.ts +++ b/packages/nx/src/utils/git-utils.ts @@ -2,8 +2,6 @@ import { exec, ExecOptions, execSync, ExecSyncOptions } from 'child_process'; import { logger } from '../devkit-exports'; import { dirname, join } from 'path'; -const SQUASH_EDITOR = join(__dirname, 'squash.js'); - function execAsync(command: string, execOptions: ExecOptions) { return new Promise((res, rej) => { exec(command, execOptions, (err, stdout, stderr) => { @@ -18,10 +16,14 @@ function execAsync(command: string, execOptions: ExecOptions) { export async function cloneFromUpstream( url: string, destination: string, - { originName } = { originName: 'origin' } + { originName, depth }: { originName: string; depth?: number } = { + originName: 'origin', + } ) { await execAsync( - `git clone ${url} ${destination} --depth 1 --origin ${originName}`, + `git clone ${url} ${destination} ${ + depth ? `--depth ${depth}` : '' + } --origin ${originName}`, { cwd: dirname(destination), } @@ -42,8 +44,8 @@ export class GitRepository { .trim(); } - addFetchRemote(remoteName: string, branch: string) { - return this.execAsync( + async addFetchRemote(remoteName: string, branch: string) { + return await this.execAsync( `git config --add remote.${remoteName}.fetch "+refs/heads/${branch}:refs/remotes/${remoteName}/${branch}"` ); } @@ -79,22 +81,16 @@ export class GitRepository { } async reset(ref: string) { - return this.execAsync(`git reset ${ref} --hard`); - } - - async squashLastTwoCommits() { - return this.execAsync( - `git -c core.editor="node ${SQUASH_EDITOR}" rebase --interactive --no-autosquash HEAD~2` - ); + return await this.execAsync(`git reset ${ref} --hard`); } async mergeUnrelatedHistories(ref: string, message: string) { - return this.execAsync( + return await this.execAsync( `git merge ${ref} -X ours --allow-unrelated-histories -m "${message}"` ); } async fetch(remote: string, ref?: string) { - return this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`); + return await this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`); } async checkout( @@ -104,7 +100,7 @@ export class GitRepository { base: string; } ) { - return this.execAsync( + return await this.execAsync( `git checkout ${opts.new ? '-b ' : ' '}${branch}${ opts.base ? ' ' + opts.base : '' }` @@ -112,30 +108,34 @@ export class GitRepository { } async move(path: string, destination: string) { - return this.execAsync(`git mv "${path}" "${destination}"`); + return await this.execAsync(`git mv "${path}" "${destination}"`); } async push(ref: string, remoteName: string) { - return this.execAsync(`git push -u -f ${remoteName} ${ref}`); + return await this.execAsync(`git push -u -f ${remoteName} ${ref}`); } async commit(message: string) { - return this.execAsync(`git commit -am "${message}"`); + return await this.execAsync(`git commit -am "${message}"`); } async amendCommit() { - return this.execAsync(`git commit --amend -a --no-edit`); + return await this.execAsync(`git commit --amend -a --no-edit`); } - deleteGitRemote(name: string) { - return this.execAsync(`git remote rm ${name}`); + async deleteGitRemote(name: string) { + return await this.execAsync(`git remote rm ${name}`); } - deleteBranch(branch: string) { - return this.execAsync(`git branch -D ${branch}`); + async addGitRemote(name: string, url: string) { + return await this.execAsync(`git remote add ${name} ${url}`); } - addGitRemote(name: string, url: string) { - return this.execAsync(`git remote add ${name} ${url}`); + async filterBranch(subdirectory: string, branchName: string) { + // We need non-ASCII file names to not be quoted, or else filter-branch will exclude them. + await this.execAsync(`git config core.quotepath false`); + return await this.execAsync( + `git filter-branch --subdirectory-filter ${subdirectory} -- ${branchName}` + ); } } @@ -150,14 +150,6 @@ export function updateRebaseFile(contents: string): string { return lines.join('\n'); } -export function fetchGitRemote( - name: string, - branch: string, - execOptions: ExecSyncOptions -) { - return execSync(`git fetch ${name} ${branch} --depth 1`, execOptions); -} - /** * This is currently duplicated in Nx Console. Please let @MaxKless know if you make changes here. */ diff --git a/packages/nx/src/utils/squash.ts b/packages/nx/src/utils/squash.ts deleted file mode 100644 index f565c068e0dc7..0000000000000 --- a/packages/nx/src/utils/squash.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { readFileSync, writeFileSync } from 'fs'; -import { updateRebaseFile } from './git-utils'; - -// This script is used as an editor for git rebase -i - -// This is the file which git creates. When this script exits, the updates should be written to this file. -const filePath = process.argv[2]; - -// Change the second commit from pick to fixup -const contents = readFileSync(filePath).toString(); -const newContents = updateRebaseFile(contents); - -// Write the updated contents back to the file -writeFileSync(filePath, newContents); From bbed59599357843bd8795c4cb8034ada7b58be1f Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Tue, 27 Aug 2024 16:15:26 -0400 Subject: [PATCH 02/12] fix(core): use git-filter-repo if it is installed --- .../src/command-line/import/command-object.ts | 2 +- packages/nx/src/command-line/import/import.ts | 2 +- .../import/utils/prepare-source-repo.ts | 15 +++++++++++---- packages/nx/src/utils/git-utils.ts | 17 +++++++++++++++++ 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts index 0914ecbdd1c1d..60b9290dd6985 100644 --- a/packages/nx/src/command-line/import/command-object.ts +++ b/packages/nx/src/command-line/import/command-object.ts @@ -5,7 +5,7 @@ import { handleErrors } from '../../utils/params'; export const yargsImportCommand: CommandModule = { command: 'import [sourceRemoteUrl] [destination]', - describe: false, + describe: `Import a project into the current workspace. Install git-filter-repo for faster imports (https://github.com/newren/git-filter-repo).`, builder: (yargs) => linkToNxDevAndExamples( withVerbose( diff --git a/packages/nx/src/command-line/import/import.ts b/packages/nx/src/command-line/import/import.ts index 2bcc4cbd29841..88a3acdf15353 100644 --- a/packages/nx/src/command-line/import/import.ts +++ b/packages/nx/src/command-line/import/import.ts @@ -94,7 +94,7 @@ export async function importHandler(options: ImportOptions) { const sourceRepoPath = join(tempImportDirectory, 'repo'); const spinner = createSpinner( - `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath}` + `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath} (Hint: use --depth to limit the clone depth for faster clone times)` ).start(); try { await rm(tempImportDirectory, { recursive: true }); diff --git a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts index f416b77abe13d..021f19527b197 100644 --- a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts +++ b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts @@ -33,10 +33,17 @@ export async function prepareSourceRepo( ); if (relativeSourceDir !== '') { - spinner.start( - `Filtering git history to only include files in ${relativeSourceDir}` - ); - await gitClient.filterBranch(relativeSourceDir, tempImportBranch); + if (await gitClient.hasFilterRepoInstalled()) { + spinner.start( + `Filtering git history to only include files in ${relativeSourceDir}` + ); + await gitClient.filterRepo(relativeSourceDir); + } else { + spinner.start( + `Filtering git history to only include files in ${relativeSourceDir} (this might take a few minutes -- install git-filter-repo for faster performance)` + ); + await gitClient.filterBranch(relativeSourceDir, tempImportBranch); + } spinner.succeed( `Filtered git history to only include files in ${relativeSourceDir}` ); diff --git a/packages/nx/src/utils/git-utils.ts b/packages/nx/src/utils/git-utils.ts index 1d58d3e90570d..3aa90f0571e81 100644 --- a/packages/nx/src/utils/git-utils.ts +++ b/packages/nx/src/utils/git-utils.ts @@ -130,6 +130,23 @@ export class GitRepository { return await this.execAsync(`git remote add ${name} ${url}`); } + async hasFilterRepoInstalled() { + try { + await this.execAsync(`git filter-repo --help`); + return true; + } catch { + return false; + } + } + + // git-filter-repo is much faster than filter-branch, but needs to be installed by user + // Use `hasFilterRepoInstalled` to check if it's installed + async filterRepo(subdirectory: string) { + return await this.execAsync( + `git filter-repo -f --subdirectory-filter ${subdirectory}` + ); + } + async filterBranch(subdirectory: string, branchName: string) { // We need non-ASCII file names to not be quoted, or else filter-branch will exclude them. await this.execAsync(`git config core.quotepath false`); From 0da6f6365347a46aaaaea99b2d634ec00b9c4c8f Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Tue, 27 Aug 2024 16:21:21 -0400 Subject: [PATCH 03/12] fix(core): nx import uses single-quotes (') in commands to avoid interpolation --- packages/nx/src/utils/git-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nx/src/utils/git-utils.ts b/packages/nx/src/utils/git-utils.ts index 3aa90f0571e81..c1740f12664e3 100644 --- a/packages/nx/src/utils/git-utils.ts +++ b/packages/nx/src/utils/git-utils.ts @@ -108,7 +108,7 @@ export class GitRepository { } async move(path: string, destination: string) { - return await this.execAsync(`git mv "${path}" "${destination}"`); + return await this.execAsync(`git mv '${path}' '${destination}'`); } async push(ref: string, remoteName: string) { From 18b65889857f88f1eed71bda108f2b7b9a2a3306 Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Tue, 27 Aug 2024 16:39:17 -0400 Subject: [PATCH 04/12] fix(core): update docs --- packages/nx/src/command-line/import/command-object.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts index 60b9290dd6985..0914ecbdd1c1d 100644 --- a/packages/nx/src/command-line/import/command-object.ts +++ b/packages/nx/src/command-line/import/command-object.ts @@ -5,7 +5,7 @@ import { handleErrors } from '../../utils/params'; export const yargsImportCommand: CommandModule = { command: 'import [sourceRemoteUrl] [destination]', - describe: `Import a project into the current workspace. Install git-filter-repo for faster imports (https://github.com/newren/git-filter-repo).`, + describe: false, builder: (yargs) => linkToNxDevAndExamples( withVerbose( From cc7682b5f1106ce2cfa8e4813eb1e8e8baace49f Mon Sep 17 00:00:00 2001 From: Jack Hsu Date: Wed, 28 Aug 2024 11:39:00 -0400 Subject: [PATCH 05/12] fix(core): handle special characters when listing and moving files --- .../import/utils/prepare-source-repo.ts | 16 ------ packages/nx/src/utils/git-utils.spec.ts | 45 --------------- packages/nx/src/utils/git-utils.ts | 57 ++++++++++++------- 3 files changed, 35 insertions(+), 83 deletions(-) diff --git a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts index 021f19527b197..b8858e9e98bc8 100644 --- a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts +++ b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts @@ -60,12 +60,7 @@ export async function prepareSourceRepo( }); } catch {} await mkdir(destinationInSource, { recursive: true }); - const gitignores = new Set(); for (const file of files) { - if (basename(file) === '.gitignore') { - gitignores.add(file); - continue; - } spinner.start( `Moving files and git history to ${destinationInSource}: ${file}` ); @@ -85,19 +80,8 @@ export async function prepareSourceRepo( `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` ); - for (const gitignore of gitignores) { - await gitClient.move(gitignore, join(destinationInSource, gitignore)); - } - await gitClient.amendCommit(); - for (const gitignore of gitignores) { - await copyFile( - join(destinationInSource, gitignore), - join(gitClient.root, gitignore) - ); - } - spinner.succeed( `${sourceRemoteUrl} has been prepared to be imported into this workspace on a temporary branch: ${tempImportBranch} in ${gitClient.root}` ); diff --git a/packages/nx/src/utils/git-utils.spec.ts b/packages/nx/src/utils/git-utils.spec.ts index e7de503b58d05..67bc43ad2b2fd 100644 --- a/packages/nx/src/utils/git-utils.spec.ts +++ b/packages/nx/src/utils/git-utils.spec.ts @@ -1,7 +1,6 @@ import { extractUserAndRepoFromGitHubUrl, getGithubSlugOrNull, - updateRebaseFile, } from './git-utils'; import { execSync } from 'child_process'; @@ -221,48 +220,4 @@ describe('git utils tests', () => { }); }); }); - - describe('updateRebaseFile', () => { - let rebaseFileContents; - - beforeEach(() => { - rebaseFileContents = `pick 6a642190 chore(repo): hi -pick 022528d9 chore(repo): prepare for import -pick 84ef7741 feat(repo): complete import of git@github.com:FrozenPandaz/created-vite-app.git - -# Rebase 3441f39e..84ef7741 onto 3441f39e (3 commands) -# -# Commands: -# p, pick = use commit -# r, reword = use commit, but edit the commit message -# e, edit = use commit, but stop for amending -# s, squash = use commit, but meld into previous commit -# f, fixup [-C | -c] = like "squash" but keep only the previous -# commit's log message, unless -C is used, in which case -# keep only this commit's message; -c is same as -C but -# opens the editor -# x, exec = run command (the rest of the line) using shell -# b, break = stop here (continue rebase later with 'git rebase --continue') -# d, drop = remove commit -# l, label