From ebf8ad0db96ad5004b55b673f0fab6c492f03b28 Mon Sep 17 00:00:00 2001 From: FrozenPandaz Date: Thu, 4 Jul 2024 18:11:20 -0400 Subject: [PATCH] feat(core): introduce nx import --- docs/generated/cli/import.md | 54 +++ docs/generated/manifests/menus.json | 8 + docs/generated/manifests/nx-api.json | 11 + docs/generated/packages-metadata.json | 11 + .../generated/packages/nx/documents/import.md | 54 +++ docs/map.json | 5 + docs/shared/reference/sitemap.md | 1 + .../src/command-line/import/command-object.ts | 39 +++ packages/nx/src/command-line/import/import.ts | 313 ++++++++++++++++++ packages/nx/src/command-line/nx-commands.ts | 2 + .../yargs-utils/shared-options.ts | 2 +- .../__snapshots__/git-utils.spec.ts.snap | 37 +++ packages/nx/src/utils/git-utils.spec.ts | 135 +++++--- packages/nx/src/utils/git-utils.ts | 178 +++++++++- packages/nx/src/utils/squash.ts | 14 + 15 files changed, 817 insertions(+), 47 deletions(-) create mode 100644 docs/generated/cli/import.md create mode 100644 docs/generated/packages/nx/documents/import.md create mode 100644 packages/nx/src/command-line/import/command-object.ts create mode 100644 packages/nx/src/command-line/import/import.ts create mode 100644 packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap create mode 100644 packages/nx/src/utils/squash.ts diff --git a/docs/generated/cli/import.md b/docs/generated/cli/import.md new file mode 100644 index 00000000000000..5f918aa08057dd --- /dev/null +++ b/docs/generated/cli/import.md @@ -0,0 +1,54 @@ +--- +title: 'import - CLI command' +description: 'Import another project into the workspace' +--- + +# import + +Import another project into the workspace + +## Usage + +```shell +nx import [sourceRemoteUrl] [destination] +``` + +Install `nx` globally to invoke the command directly using `nx`, or use `npx nx`, `yarn nx`, or `pnpm nx`. + +## Options + +### destination + +Type: `string` + +The destination in the current workspace + +### help + +Type: `boolean` + +Show help + +### ref + +Type: `string` + +The branch to import + +### sourceRemoteUrl + +Type: `string` + +The remote URL of the source to import + +### verbose + +Type: `boolean` + +Prints additional information about the commands (e.g., stack traces) + +### version + +Type: `boolean` + +Show version number diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index e09f52d326ecd8..4bce99d85d01b7 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -8544,6 +8544,14 @@ "isExternal": false, "children": [], "disableCollapsible": false + }, + { + "name": "import", + "path": "/nx-api/nx/documents/import", + "id": "import", + "isExternal": false, + "children": [], + "disableCollapsible": false } ], "isExternal": false, diff --git a/docs/generated/manifests/nx-api.json b/docs/generated/manifests/nx-api.json index a6ea31c52f2066..b793c77c62a6d2 100644 --- a/docs/generated/manifests/nx-api.json +++ b/docs/generated/manifests/nx-api.json @@ -1949,6 +1949,17 @@ "path": "/nx-api/nx/documents/add", "tags": [], "originalFilePath": "generated/cli/add" + }, + "/nx-api/nx/documents/import": { + "id": "import", + "name": "import", + "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", + "file": "generated/packages/nx/documents/import", + "itemList": [], + "isExternal": false, + "path": "/nx-api/nx/documents/import", + "tags": [], + "originalFilePath": "generated/cli/import" } }, "root": "/packages/nx", diff --git a/docs/generated/packages-metadata.json b/docs/generated/packages-metadata.json index 9d2d075fd1524c..358ba2a54cfad0 100644 --- a/docs/generated/packages-metadata.json +++ b/docs/generated/packages-metadata.json @@ -1928,6 +1928,17 @@ "path": "nx/documents/add", "tags": [], "originalFilePath": "generated/cli/add" + }, + { + "id": "import", + "name": "import", + "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", + "file": "generated/packages/nx/documents/import", + "itemList": [], + "isExternal": false, + "path": "nx/documents/import", + "tags": [], + "originalFilePath": "generated/cli/import" } ], "executors": [ diff --git a/docs/generated/packages/nx/documents/import.md b/docs/generated/packages/nx/documents/import.md new file mode 100644 index 00000000000000..5f918aa08057dd --- /dev/null +++ b/docs/generated/packages/nx/documents/import.md @@ -0,0 +1,54 @@ +--- +title: 'import - CLI command' +description: 'Import another project into the workspace' +--- + +# import + +Import another project into the workspace + +## Usage + +```shell +nx import [sourceRemoteUrl] [destination] +``` + +Install `nx` globally to invoke the command directly using `nx`, or use `npx nx`, `yarn nx`, or `pnpm nx`. + +## Options + +### destination + +Type: `string` + +The destination in the current workspace + +### help + +Type: `boolean` + +Show help + +### ref + +Type: `string` + +The branch to import + +### sourceRemoteUrl + +Type: `string` + +The remote URL of the source to import + +### verbose + +Type: `boolean` + +Prints additional information about the commands (e.g., stack traces) + +### version + +Type: `boolean` + +Show version number diff --git a/docs/map.json b/docs/map.json index 8908a255340069..cf47e2ceccc3b7 100644 --- a/docs/map.json +++ b/docs/map.json @@ -2075,6 +2075,11 @@ "name": "add", "id": "add", "file": "generated/cli/add" + }, + { + "name": "import", + "id": "import", + "file": "generated/cli/import" } ] }, diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index 631bf228a2f2d8..28d2fed52bb720 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -548,6 +548,7 @@ - [view-logs](/nx-api/nx/documents/view-logs) - [release](/nx-api/nx/documents/release) - [add](/nx-api/nx/documents/add) + - [import](/nx-api/nx/documents/import) - [executors](/nx-api/nx/executors) - [noop](/nx-api/nx/executors/noop) - [run-commands](/nx-api/nx/executors/run-commands) diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts new file mode 100644 index 00000000000000..ca2272776d644e --- /dev/null +++ b/packages/nx/src/command-line/import/command-object.ts @@ -0,0 +1,39 @@ +import { Arguments, CommandModule, MiddlewareFunction } from 'yargs'; +import { linkToNxDevAndExamples } from '../yargs-utils/documentation'; +import { withVerbose } from '../yargs-utils/shared-options'; +import { handleErrors } from '../../utils/params'; +import type { ImportOptions } from './import'; + +export const yargsImportCommand: CommandModule = { + command: 'import [sourceRemoteUrl] [destination]', + describe: 'Import another project into the workspace', + builder: (yargs) => + linkToNxDevAndExamples( + withVerbose( + yargs + .positional('sourceRemoteUrl', { + type: 'string', + description: 'The remote URL of the source to import', + }) + .positional('destination', { + type: 'string', + description: 'The destination in the current workspace', + }) + .option('ref', { + type: 'string', + description: 'The branch to import', + }) + .requiresArg('ref') + ), + 'import' + ), + handler: async (args) => { + const exitCode = await handleErrors( + (args.verbose as boolean) ?? process.env.NX_VERBOSE_LOGGING === 'true', + async () => { + return (await import('./import')).importHandler(args as any); + } + ); + process.exit(exitCode); + }, +}; diff --git a/packages/nx/src/command-line/import/import.ts b/packages/nx/src/command-line/import/import.ts new file mode 100644 index 00000000000000..a37e4eb9a28caf --- /dev/null +++ b/packages/nx/src/command-line/import/import.ts @@ -0,0 +1,313 @@ +import { basename, dirname, join, relative } from 'path'; +import { cloneFromUpstream, GitRepository } from '../../utils/git-utils'; +import { copyFile, mkdir, readdir, rm } from 'fs'; +import { promisify } from 'util'; +import { tmpdir } from 'tmp'; +import { prompt } from 'enquirer'; +import { output } from '../../utils/output'; +import * as createSpinner from 'ora'; + +const readdirAsync = promisify(readdir); +const rmAsync = promisify(rm); +const copyFileAsync = promisify(copyFile); +const mkdirAsync = promisify(mkdir); + +function wait(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export interface ImportOptions { + sourceRemoteUrl: string; + ref: string; + destination: string; + verbose: boolean; + interactive: boolean; +} + +const importRemoteName = '__tmp_nx_import__'; +const tempImportBranch = '__tmp_import_stage__'; +const tempFileDir = '__tmp_import_stage__'; + +async function prepareSourceRepo( + gitClient: GitRepository, + ref: string, + sourceDir: string, + tempSourceDir: string +) { + const spinner = createSpinner(`Checking out ${ref}`).start(); + await gitClient.checkout(tempImportBranch, { + new: true, + base: `origin/${ref}`, + }); + spinner.succeed(`Checked out ${ref}`); + const relativeSourceDir = relative(gitClient.root, sourceDir); + + spinner.start(`Preparing for import`); + if (relativeSourceDir === '') { + try { + await rmAsync(tempSourceDir, { + recursive: true, + }); + } catch {} + const files = await gitClient.getGitFiles('.'); + // const entries = await readdirAsync(gitClient.repoRoot); + await mkdirAsync(tempSourceDir); + const gitignores = new Set(); + for (const file of files) { + if (basename(file) === '.gitignore') { + gitignores.add(file); + continue; + } + + await mkdirAsync(dirname(join(tempSourceDir, file)), { recursive: true }); + + // await wait(25); + + const newPath = join(tempSourceDir, file); + // console.log('Moving', file, 'to', newPath); + try { + await gitClient.move(file, newPath); + } catch { + // console.log('failed once'); + await wait(100); + await gitClient.move(file, newPath); + } + } + + await gitClient.stageToGit(tempSourceDir); + await gitClient.commit('chore(repo): prepare for import'); + + for (const gitignore of gitignores) { + await gitClient.move(gitignore, join(tempSourceDir, gitignore)); + await copyFileAsync( + join(tempSourceDir, gitignore), + join(gitClient.root, gitignore) + ); + } + await gitClient.stageToGit('.'); + await gitClient.commit('chore(repo): prepare for import 2'); + await gitClient.squashLastTwoCommits(); + } else { + throw new Error('boom'); + + await gitClient.move(sourceDir, join(tempSourceDir, relativeSourceDir)); + + const entries = await readdirAsync(gitClient.root); + await mkdirAsync(tempSourceDir); + for (const entry of entries) { + if (entry === '.gitignore') { + continue; + } + + await gitClient.move(entry, join('garbage', entry)); + } + + await gitClient.stageToGit(tempSourceDir, 'garbage'); + } + spinner.succeed(`Prepared for import`); + + // Move our source directory into a temp directory + // await Promise.all( + // entries.map(async (entry) => { + // await renameAsync(entry, tempSourceDir); + // }) + // ); + + // Delete everything else to avoid conflicts + // const otherEntries = await readdirAsync(sourceDir); + // await Promise.all( + // otherEntries + // .filter((entry) => {}) + // .map(async (entry) => { + // await rmAsync(entry); + // }) + // ); +} + +async function confirmOrExitWithAnError(message: string) { + const { confirm } = await prompt<{ confirm: boolean }>([ + { + type: 'confirm', + name: 'confirm', + message, + }, + ]); + + if (confirm === false) { + throw new Error('Cancelled'); + } +} + +async function mergeRemoteSource( + destinationGitClient: GitRepository, + sourceRemoteUrl: string, + branch: string, + destination: string +) { + const spinner = createSpinner(); + spinner.start(`Merged ${branch} from ${sourceRemoteUrl} into ${destination}`); + + spinner.start(`Adding ${sourceRemoteUrl} as a remote`); + await createTemporarySourceRemote( + destinationGitClient, + sourceRemoteUrl, + importRemoteName + ); + spinner.succeed(`Added ${sourceRemoteUrl} as a remote`); + + spinner.start( + `Merging files and git history from ${branch} from ${sourceRemoteUrl} into a temporary directory` + ); + await destinationGitClient.mergeUnrelatedHistories( + `${importRemoteName}/${branch}`, + `feat(repo): merge ${sourceRemoteUrl}` + ); + spinner.succeed( + `Merging files and git history from ${branch} from ${sourceRemoteUrl} into a temporary directory` + ); + spinner.start( + `Moving files and git history a temporary directory to ${destination}` + ); + + await mkdirAsync(destination, { recursive: true }); + + const files = await destinationGitClient.getGitFiles(tempFileDir); + + for (const file of files) { + const newPath = join(destination, relative(tempFileDir, file)); + + await mkdirAsync(dirname(newPath), { recursive: true }); + + try { + await destinationGitClient.move(file, newPath); + } catch { + console.log('failed once'); + await wait(100); + await destinationGitClient.move(file, newPath); + } + } + spinner.succeed( + `Moved files and git history a temporary directory to ${destination}` + ); + spinner.start(`Committing changes`); + + await rmAsync(join(destinationGitClient.root, tempFileDir), { + recursive: true, + }); + + await destinationGitClient.commit( + `feat(repo): complete import of ${sourceRemoteUrl}` + ); + await destinationGitClient.squashLastTwoCommits(); + + spinner.succeed(`Committed changes`); + spinner.succeed( + `Merged ${branch} from ${sourceRemoteUrl} into ${destination}` + ); +} + +async function createTemporarySourceRemote( + destinationGitClient: GitRepository, + sourceRemoteUrl: string, + remoteName: string +) { + try { + await destinationGitClient.deleteGitRemote(remoteName); + } catch {} + await destinationGitClient.addGitRemote(remoteName, sourceRemoteUrl); + await destinationGitClient.fetch(remoteName); +} + +export async function importHandler(options: ImportOptions) { + let { sourceRemoteUrl, ref, destination } = options; + + output.log({ + title: + 'Nx will walk you through the process of importing another repo into the workspace:', + }); + + const tempRepoPath = join(tmpdir, 'nx-import'); + const absSource = join(tempRepoPath, 'repo'); + + if (!sourceRemoteUrl) { + sourceRemoteUrl = ( + await prompt<{ sourceRemoteUrl: string }>([ + { + type: 'input', + name: 'sourceRemoteUrl', + message: + 'What is the Remote URL of the repository you want to import? Nx will clone this to a temporary directory', + required: true, + }, + ]) + ).sourceRemoteUrl; + } + + const spinner = createSpinner(`Cloning ${sourceRemoteUrl}`).start(); + try { + await rmAsync(tempRepoPath, { recursive: true }); + } catch {} + await mkdirAsync(tempRepoPath, { recursive: true }); + // await confirmOrExitWithAnError( + // `Clone repo into a temporary directory where it will be prepared to import, ${tempRepoPath} from ${sourceRemoteUrl}` + // ); + const sourceGitClient = await cloneFromUpstream( + sourceRemoteUrl, + join(tempRepoPath, 'repo') + ); + spinner.succeed(`Cloned ${sourceRemoteUrl}`); + + if (!ref) { + const branchChoices = await sourceGitClient.listBranches(); + ref = ( + await prompt<{ ref: string }>([ + { + type: 'autocomplete', + name: 'ref', + message: + 'Which branch do you want to import? Nx will prepare it for importing it into this workspace', + choices: branchChoices, + required: true, + }, + ]) + ).ref; + } + + await wait(100); + + const tempSourceDir = join(sourceGitClient.root, tempFileDir); + await prepareSourceRepo(sourceGitClient, ref, absSource, tempSourceDir); + + console.log(await sourceGitClient.showStat()); + + await confirmOrExitWithAnError( + `The repo has been prepared, see the changes above. Push the changes to ${sourceRemoteUrl} (git push -u -f origin ${tempImportBranch})` + ); + await sourceGitClient.push(tempImportBranch); + + // Ready to import + const destinationGitClient = new GitRepository(process.cwd()); + + if (!destination) { + destination = ( + await prompt<{ destination: string }>([ + { + type: 'input', + name: 'destination', + message: 'Which directory do you want to import the project into?', + required: true, + }, + ]) + ).destination; + } + + await confirmOrExitWithAnError( + `Importing the changes into this repo into a temporary directory (git merge ${importRemoteName}/${tempImportBranch} -X ours --allow-unrelated-histories)` + ); + await mergeRemoteSource( + destinationGitClient, + sourceRemoteUrl, + tempImportBranch, + destination + ); +} diff --git a/packages/nx/src/command-line/nx-commands.ts b/packages/nx/src/command-line/nx-commands.ts index 6ba7000c617815..78ac8a484c2c23 100644 --- a/packages/nx/src/command-line/nx-commands.ts +++ b/packages/nx/src/command-line/nx-commands.ts @@ -20,6 +20,7 @@ import { yargsFormatWriteCommand, } from './format/command-object'; import { yargsGenerateCommand } from './generate/command-object'; +import { yargsImportCommand } from './import/command-object'; import { yargsInitCommand } from './init/command-object'; import { yargsListCommand } from './list/command-object'; import { @@ -73,6 +74,7 @@ export const commandsObject = yargs .command(yargsFormatCheckCommand) .command(yargsFormatWriteCommand) .command(yargsGenerateCommand) + .command(yargsImportCommand) .command(yargsInitCommand) .command(yargsInternalMigrateCommand) .command(yargsListCommand) diff --git a/packages/nx/src/command-line/yargs-utils/shared-options.ts b/packages/nx/src/command-line/yargs-utils/shared-options.ts index 3196074425adfc..bd9289a4114710 100644 --- a/packages/nx/src/command-line/yargs-utils/shared-options.ts +++ b/packages/nx/src/command-line/yargs-utils/shared-options.ts @@ -118,7 +118,7 @@ export function withConfiguration(yargs: Argv) { }); } -export function withVerbose(yargs: Argv) { +export function withVerbose(yargs: Argv) { return yargs .option('verbose', { describe: diff --git a/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap b/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap new file mode 100644 index 00000000000000..4f4ebae3ef1757 --- /dev/null +++ b/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap @@ -0,0 +1,37 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`git utils updateRebaseFile should squash the last 2 commits 1`] = ` +"pick 6a642190 chore(repo): hi +pick 022528d9 chore(repo): prepare for import +fixup 84ef7741 feat(repo): complete import of git@github.com:FrozenPandaz/created-vite-app.git + +# Rebase 3441f39e..84ef7741 onto 3441f39e (3 commands) +# +# Commands: +# p, pick = use commit +# r, reword = use commit, but edit the commit message +# e, edit = use commit, but stop for amending +# s, squash = use commit, but meld into previous commit +# f, fixup [-C | -c] = like "squash" but keep only the previous +# commit's log message, unless -C is used, in which case +# keep only this commit's message; -c is same as -C but +# opens the editor +# x, exec = run command (the rest of the line) using shell +# b, break = stop here (continue rebase later with 'git rebase --continue') +# d, drop = remove commit +# l, label