From 2b7ec6cab3ab66a0bf772bf9c9b0d49c719f20d7 Mon Sep 17 00:00:00 2001 From: FrozenPandaz Date: Thu, 4 Jul 2024 18:11:20 -0400 Subject: [PATCH] feat(core): introduce nx import --- docs/generated/cli/import.md | 54 +++++ docs/generated/manifests/menus.json | 8 + docs/generated/manifests/nx-api.json | 11 + docs/generated/packages-metadata.json | 11 + .../generated/packages/nx/documents/import.md | 54 +++++ docs/map.json | 5 + docs/shared/reference/sitemap.md | 1 + .../src/command-line/import/command-object.ts | 38 +++ packages/nx/src/command-line/import/import.ts | 224 ++++++++++++++++++ packages/nx/src/command-line/nx-commands.ts | 2 + .../yargs-utils/shared-options.ts | 2 +- packages/nx/src/utils/git-utils.ts | 134 ++++++++++- 12 files changed, 542 insertions(+), 2 deletions(-) create mode 100644 docs/generated/cli/import.md create mode 100644 docs/generated/packages/nx/documents/import.md create mode 100644 packages/nx/src/command-line/import/command-object.ts create mode 100644 packages/nx/src/command-line/import/import.ts diff --git a/docs/generated/cli/import.md b/docs/generated/cli/import.md new file mode 100644 index 0000000000000..5f918aa08057d --- /dev/null +++ b/docs/generated/cli/import.md @@ -0,0 +1,54 @@ +--- +title: 'import - CLI command' +description: 'Import another project into the workspace' +--- + +# import + +Import another project into the workspace + +## Usage + +```shell +nx import [sourceRemoteUrl] [destination] +``` + +Install `nx` globally to invoke the command directly using `nx`, or use `npx nx`, `yarn nx`, or `pnpm nx`. + +## Options + +### destination + +Type: `string` + +The destination in the current workspace + +### help + +Type: `boolean` + +Show help + +### ref + +Type: `string` + +The branch to import + +### sourceRemoteUrl + +Type: `string` + +The remote URL of the source to import + +### verbose + +Type: `boolean` + +Prints additional information about the commands (e.g., stack traces) + +### version + +Type: `boolean` + +Show version number diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index e09f52d326ecd..4bce99d85d01b 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -8544,6 +8544,14 @@ "isExternal": false, "children": [], "disableCollapsible": false + }, + { + "name": "import", + "path": "/nx-api/nx/documents/import", + "id": "import", + "isExternal": false, + "children": [], + "disableCollapsible": false } ], "isExternal": false, diff --git a/docs/generated/manifests/nx-api.json b/docs/generated/manifests/nx-api.json index a6ea31c52f206..b793c77c62a6d 100644 --- a/docs/generated/manifests/nx-api.json +++ b/docs/generated/manifests/nx-api.json @@ -1949,6 +1949,17 @@ "path": "/nx-api/nx/documents/add", "tags": [], "originalFilePath": "generated/cli/add" + }, + "/nx-api/nx/documents/import": { + "id": "import", + "name": "import", + "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", + "file": "generated/packages/nx/documents/import", + "itemList": [], + "isExternal": false, + "path": "/nx-api/nx/documents/import", + "tags": [], + "originalFilePath": "generated/cli/import" } }, "root": "/packages/nx", diff --git a/docs/generated/packages-metadata.json b/docs/generated/packages-metadata.json index 9d2d075fd1524..358ba2a54cfad 100644 --- a/docs/generated/packages-metadata.json +++ b/docs/generated/packages-metadata.json @@ -1928,6 +1928,17 @@ "path": "nx/documents/add", "tags": [], "originalFilePath": "generated/cli/add" + }, + { + "id": "import", + "name": "import", + "description": "The core Nx plugin contains the core functionality of Nx like the project graph, nx commands and task orchestration.", + "file": "generated/packages/nx/documents/import", + "itemList": [], + "isExternal": false, + "path": "nx/documents/import", + "tags": [], + "originalFilePath": "generated/cli/import" } ], "executors": [ diff --git a/docs/generated/packages/nx/documents/import.md b/docs/generated/packages/nx/documents/import.md new file mode 100644 index 0000000000000..5f918aa08057d --- /dev/null +++ b/docs/generated/packages/nx/documents/import.md @@ -0,0 +1,54 @@ +--- +title: 'import - CLI command' +description: 'Import another project into the workspace' +--- + +# import + +Import another project into the workspace + +## Usage + +```shell +nx import [sourceRemoteUrl] [destination] +``` + +Install `nx` globally to invoke the command directly using `nx`, or use `npx nx`, `yarn nx`, or `pnpm nx`. + +## Options + +### destination + +Type: `string` + +The destination in the current workspace + +### help + +Type: `boolean` + +Show help + +### ref + +Type: `string` + +The branch to import + +### sourceRemoteUrl + +Type: `string` + +The remote URL of the source to import + +### verbose + +Type: `boolean` + +Prints additional information about the commands (e.g., stack traces) + +### version + +Type: `boolean` + +Show version number diff --git a/docs/map.json b/docs/map.json index 8908a25534006..cf47e2ceccc3b 100644 --- a/docs/map.json +++ b/docs/map.json @@ -2075,6 +2075,11 @@ "name": "add", "id": "add", "file": "generated/cli/add" + }, + { + "name": "import", + "id": "import", + "file": "generated/cli/import" } ] }, diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index 631bf228a2f2d..28d2fed52bb72 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -548,6 +548,7 @@ - [view-logs](/nx-api/nx/documents/view-logs) - [release](/nx-api/nx/documents/release) - [add](/nx-api/nx/documents/add) + - [import](/nx-api/nx/documents/import) - [executors](/nx-api/nx/executors) - [noop](/nx-api/nx/executors/noop) - [run-commands](/nx-api/nx/executors/run-commands) diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts new file mode 100644 index 0000000000000..5ed38f0a50c83 --- /dev/null +++ b/packages/nx/src/command-line/import/command-object.ts @@ -0,0 +1,38 @@ +import { CommandModule } from 'yargs'; +import { linkToNxDevAndExamples } from '../yargs-utils/documentation'; +import { withVerbose } from '../yargs-utils/shared-options'; +import { handleErrors } from '../../utils/params'; + +export const yargsImportCommand: CommandModule = { + command: 'import [sourceRemoteUrl] [destination]', + describe: 'Import another project into the workspace', + builder: (yargs) => + linkToNxDevAndExamples( + withVerbose( + yargs + .positional('sourceRemoteUrl', { + type: 'string', + description: 'The remote URL of the source to import', + }) + .positional('destination', { + type: 'string', + description: 'The destination in the current workspace', + }) + .option('ref', { + type: 'string', + description: 'The branch to import', + }) + .requiresArg('ref') + ), + 'import' + ), + handler: async (args) => { + const exitCode = await handleErrors( + (args.verbose as boolean) ?? process.env.NX_VERBOSE_LOGGING === 'true', + async () => { + return (await import('./import')).importHandler(args as any); + } + ); + process.exit(exitCode); + }, +}; diff --git a/packages/nx/src/command-line/import/import.ts b/packages/nx/src/command-line/import/import.ts new file mode 100644 index 0000000000000..4b6077062e6fb --- /dev/null +++ b/packages/nx/src/command-line/import/import.ts @@ -0,0 +1,224 @@ +import { basename, dirname, join, relative } from 'path'; +import { cloneFromUpstream, GitClient } from '../../utils/git-utils'; +import { copyFile, mkdir, readdir, rm } from 'fs'; +import { promisify } from 'util'; +import { tmpdir } from 'tmp'; +import { prompt } from 'enquirer'; + +const readdirAsync = promisify(readdir); +const rmAsync = promisify(rm); +const copyFileAsync = promisify(copyFile); +const mkdirAsync = promisify(mkdir); + +function wait(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export interface ImportOptions { + sourceRemoteUrl: string; + ref: string; + destination: string; + verbose: boolean; +} + +const importRemoteName = '__tmp_nx_import__'; +const tempImportBranch = '__tmp_import_stage__'; +const tempFileDir = '__tmp_import_stage__'; + +async function prepareSourceRepo( + gitClient: GitClient, + sourceDir: string, + tempSourceDir: string +) { + const relativeSourceDir = relative(gitClient.root, sourceDir); + + if (relativeSourceDir === '') { + try { + await rmAsync(tempSourceDir, { + recursive: true, + }); + } catch {} + const files = await gitClient.getGitFiles('.'); + // const entries = await readdirAsync(gitClient.repoRoot); + await mkdirAsync(tempSourceDir); + const gitignores = new Set(); + for (const file of files) { + if (basename(file) === '.gitignore') { + gitignores.add(file); + continue; + } + + await mkdirAsync(dirname(join(tempSourceDir, file)), { recursive: true }); + + // await wait(25); + + const newPath = join(tempSourceDir, file); + console.log('Moving', file, 'to', newPath); + try { + await gitClient.move(file, newPath); + } catch { + console.log('failed once'); + await wait(100); + await gitClient.move(file, newPath); + } + } + + await gitClient.stageToGit(tempSourceDir); + await gitClient.commit('prepare for import'); + + for (const gitignore of gitignores) { + await gitClient.move(gitignore, join(tempSourceDir, gitignore)); + await copyFileAsync( + join(tempSourceDir, gitignore), + join(gitClient.root, gitignore) + ); + } + await gitClient.stageToGit('.'); + await gitClient.commit('prepare for import 2'); + } else { + throw new Error('boom'); + + await gitClient.move(sourceDir, join(tempSourceDir, relativeSourceDir)); + + const entries = await readdirAsync(gitClient.root); + await mkdirAsync(tempSourceDir); + for (const entry of entries) { + if (entry === '.gitignore') { + continue; + } + + await gitClient.move(entry, join('garbage', entry)); + } + + await gitClient.stageToGit(tempSourceDir, 'garbage'); + } + + // Move our source directory into a temp directory + // await Promise.all( + // entries.map(async (entry) => { + // await renameAsync(entry, tempSourceDir); + // }) + // ); + + // Delete everything else to avoid conflicts + // const otherEntries = await readdirAsync(sourceDir); + // await Promise.all( + // otherEntries + // .filter((entry) => {}) + // .map(async (entry) => { + // await rmAsync(entry); + // }) + // ); +} + +async function confirmOrExitWithAnError(message: string) { + const { confirm } = await prompt<{ confirm: boolean }>([ + { + type: 'confirm', + name: 'confirm', + message, + }, + ]); + + if (confirm === false) { + throw new Error('Cancelled'); + } +} + +export async function importHandler(options: ImportOptions) { + const { sourceRemoteUrl, ref, destination } = options; + + const tempRepoPath = join(tmpdir, 'nx-import'); + + try { + await rmAsync(tempRepoPath, { recursive: true }); + } catch {} + await mkdirAsync(tempRepoPath, { recursive: true }); + await confirmOrExitWithAnError( + `Clone repo into a temporary directory where it will be prepared to import, ${tempRepoPath} from ${sourceRemoteUrl}` + ); + await cloneFromUpstream(sourceRemoteUrl, 'repo', { + cwd: tempRepoPath, + }); + const absSource = join(tempRepoPath, 'repo'); + + const sourceGitClient = new GitClient(absSource); + + await wait(100); + + await sourceGitClient.checkout(tempImportBranch, { + new: true, + base: `origin/${ref}`, + }); + + const tempSourceDir = join(sourceGitClient.root, tempFileDir); + await prepareSourceRepo(sourceGitClient, absSource, tempSourceDir); + + await confirmOrExitWithAnError( + `Pushing prepared repo as ${tempImportBranch} to ${sourceRemoteUrl} (git push -u -f origin ${tempImportBranch})` + ); + await sourceGitClient.push(tempImportBranch); + + // Ready to import + + const destinationGitClient = new GitClient(process.cwd()); + + await confirmOrExitWithAnError( + `Adding ${sourceRemoteUrl} as a remote in this repo (git remote add ${importRemoteName} ${sourceRemoteUrl})` + ); + try { + await destinationGitClient.deleteGitRemote(importRemoteName); + } catch {} + await destinationGitClient.addGitRemote(importRemoteName, sourceRemoteUrl); + await destinationGitClient.fetch(importRemoteName); + + await confirmOrExitWithAnError( + `Importing the changes into this repo into a temporary directory (git merge ${importRemoteName}/${tempImportBranch} -X ours --allow-unrelated-histories)` + ); + await destinationGitClient.merge( + `${importRemoteName}/${tempImportBranch}`, + `feat(repo): merge ${sourceRemoteUrl}` + ); + + await mkdirAsync(destination, { recursive: true }); + + const files = await destinationGitClient.getGitFiles(tempFileDir); + + for (const file of files) { + const newPath = join(destination, relative(tempFileDir, file)); + + await mkdirAsync(dirname(newPath), { recursive: true }); + + console.log('Moving', file, 'to', newPath); + try { + await destinationGitClient.move(file, newPath); + } catch { + console.log('failed once'); + await wait(100); + await destinationGitClient.move(file, newPath); + } + } + + await destinationGitClient.commit( + `feat(repo): complete import of ${sourceRemoteUrl}` + ); + + // Ensure that tmp remote does not exist + // try { + // deleteGitRemote(importRemoteName, { + // cwd: workspaceRoot, + // }); + // } catch { + // // It's okay if it errors because it means that it did not exist. + // } + // + // // Add source remote to destination workspace + // addGitRemote(importRemoteName, sourceGitRemote.url, { + // cwd: workspaceRoot, + // }); + // + // // Fetch the remote + // fetchGitRemote(importRemoteName, tempImportBranch, { cwd: workspaceRoot }); + // + // console.log({ remotes: sourceGitRemote }); +} diff --git a/packages/nx/src/command-line/nx-commands.ts b/packages/nx/src/command-line/nx-commands.ts index 6ba7000c61781..78ac8a484c2c2 100644 --- a/packages/nx/src/command-line/nx-commands.ts +++ b/packages/nx/src/command-line/nx-commands.ts @@ -20,6 +20,7 @@ import { yargsFormatWriteCommand, } from './format/command-object'; import { yargsGenerateCommand } from './generate/command-object'; +import { yargsImportCommand } from './import/command-object'; import { yargsInitCommand } from './init/command-object'; import { yargsListCommand } from './list/command-object'; import { @@ -73,6 +74,7 @@ export const commandsObject = yargs .command(yargsFormatCheckCommand) .command(yargsFormatWriteCommand) .command(yargsGenerateCommand) + .command(yargsImportCommand) .command(yargsInitCommand) .command(yargsInternalMigrateCommand) .command(yargsListCommand) diff --git a/packages/nx/src/command-line/yargs-utils/shared-options.ts b/packages/nx/src/command-line/yargs-utils/shared-options.ts index 3196074425adf..bd9289a411471 100644 --- a/packages/nx/src/command-line/yargs-utils/shared-options.ts +++ b/packages/nx/src/command-line/yargs-utils/shared-options.ts @@ -118,7 +118,7 @@ export function withConfiguration(yargs: Argv) { }); } -export function withVerbose(yargs: Argv) { +export function withVerbose(yargs: Argv) { return yargs .option('verbose', { describe: diff --git a/packages/nx/src/utils/git-utils.ts b/packages/nx/src/utils/git-utils.ts index 2014e2c19b668..4515c6b4f5fba 100644 --- a/packages/nx/src/utils/git-utils.ts +++ b/packages/nx/src/utils/git-utils.ts @@ -1,6 +1,138 @@ -import { execSync } from 'child_process'; +import { exec, ExecOptions, execSync, ExecSyncOptions } from 'child_process'; import { logger } from '../devkit-exports'; +function execAsync(command: string, execOptions: ExecOptions) { + return new Promise((res, rej) => { + exec(command, execOptions, (err, stdout, stderr) => { + if (err) { + return rej(err); + } + res(stdout); + }); + }); +} + +export function cloneFromUpstream( + url: string, + destination: string, + execOptions: ExecOptions +) { + return execAsync(`git clone ${url} ${destination} --depth 1`, execOptions); +} + +export class GitClient { + public root = this.getGitRootPath(this.directory); + constructor(private directory: string) {} + + getGitRootPath(cwd: string) { + return execSync('git rev-parse --show-toplevel', { + cwd, + }) + .toString() + .trim(); + } + + private execAsync(command: string) { + return execAsync(command, { + cwd: this.root, + }); + } + + async getGitFiles(path: string) { + return (await this.execAsync(`git ls-files ${path}`)) + .trim() + .split('\n') + .map((s) => s.trim()); + } + + async deleteBranch(branch: string) { + return this.execAsync(`git branch -D ${branch}`); + } + + async merge(ref: string, message: string) { + return this.execAsync( + `git merge ${ref} -X ours --allow-unrelated-histories -m "${message}"` + ); + } + async fetch(remote: string) { + return this.execAsync(`git fetch ${remote}`); + } + + async checkout( + branch: string, + opts: { + new: boolean; + base: string; + } + ) { + return this.execAsync( + `git checkout ${opts.new ? '-b ' : ' '}${branch}${ + opts.base ? ' ' + opts.base : '' + }` + ); + } + + async move(path: string, destination: string) { + return this.execAsync(`git mv ${path} ${destination}`); + } + + async push(ref: string) { + return this.execAsync(`git push -u -f origin ${ref}`); + } + + async getGitRemotes(): Promise< + Array<{ + name: string; + url: string; + }> + > { + const remotes = (await this.execAsync('git remote -v')) + .toString() + .split('\n') + .filter((line) => line.endsWith(' (fetch)')) + .map((s) => s.replace(' (fetch)', '').split('\t')) + .map(([name, url]) => ({ + name, + url, + })); + + return remotes; + } + + async stageToGit(...paths: string[]) { + return this.execAsync(`git add ${paths.join(' ')}`); + } + + async commit(message: string) { + return this.execAsync(`git commit -m "${message}"`); + } + + async isIgnored(path: string) { + try { + await this.execAsync(`git check-ignore ${path}`); + return true; + } catch { + return false; + } + } + + deleteGitRemote(name: string) { + return this.execAsync(`git remote rm ${name}`); + } + + addGitRemote(name: string, url: string) { + return this.execAsync(`git remote add ${name} ${url}`); + } +} + +export function fetchGitRemote( + name: string, + branch: string, + execOptions: ExecSyncOptions +) { + return execSync(`git fetch ${name} ${branch} --depth 1`, execOptions); +} + export function getGithubSlugOrNull(): string | null { try { const gitRemote = execSync('git remote -v').toString();