diff --git a/e2e/nx/src/import.test.ts b/e2e/nx/src/import.test.ts index 2bb5196d589d1..54ad7ea3874e3 100644 --- a/e2e/nx/src/import.test.ts +++ b/e2e/nx/src/import.test.ts @@ -8,7 +8,7 @@ import { updateFile, e2eCwd, } from '@nx/e2e/utils'; -import { mkdirSync, rmdirSync } from 'fs'; +import { writeFileSync, mkdirSync, rmdirSync } from 'fs'; import { execSync } from 'node:child_process'; import { join } from 'path'; @@ -86,4 +86,54 @@ describe('Nx Import', () => { ); runCLI(`vite:build created-vite-app`); }); + + it('should be able to import two directories from same repo', () => { + // Setup repo with two packages: a and b + const repoPath = join(tempImportE2ERoot, 'repo'); + mkdirSync(repoPath, { recursive: true }); + writeFileSync(join(repoPath, 'README.md'), `# Repo`); + execSync(`git init`, { + cwd: repoPath, + }); + execSync(`git add .`, { + cwd: repoPath, + }); + execSync(`git commit -am "initial commit"`, { + cwd: repoPath, + }); + execSync(`git checkout -b main`, { + cwd: repoPath, + }); + mkdirSync(join(repoPath, 'packages/a'), { recursive: true }); + writeFileSync(join(repoPath, 'packages/a/README.md'), `# A`); + execSync(`git add packages/a`, { + cwd: repoPath, + }); + execSync(`git commit -m "add package a"`, { + cwd: repoPath, + }); + mkdirSync(join(repoPath, 'packages/b'), { recursive: true }); + writeFileSync(join(repoPath, 'packages/b/README.md'), `# B`); + execSync(`git add packages/b`, { + cwd: repoPath, + }); + execSync(`git commit -m "add package b"`, { + cwd: repoPath, + }); + + runCLI( + `import ${repoPath} packages/a --ref main --source packages/a --no-interactive`, + { + verbose: true, + } + ); + runCLI( + `import ${repoPath} packages/b --ref main --source packages/b --no-interactive`, + { + verbose: true, + } + ); + + checkFilesExist('packages/a/README.md', 'packages/b/README.md'); + }); }); diff --git a/packages/nx/src/command-line/import/command-object.ts b/packages/nx/src/command-line/import/command-object.ts index 0158249d09102..0914ecbdd1c1d 100644 --- a/packages/nx/src/command-line/import/command-object.ts +++ b/packages/nx/src/command-line/import/command-object.ts @@ -28,6 +28,11 @@ export const yargsImportCommand: CommandModule = { type: 'string', description: 'The branch from the source repository to import', }) + .option('depth', { + type: 'number', + description: + 'The depth to clone the source repository (limit this for faster git clone)', + }) .option('interactive', { type: 'boolean', description: 'Interactive mode', diff --git a/packages/nx/src/command-line/import/import.ts b/packages/nx/src/command-line/import/import.ts index ee120e2d88487..3d425c1226424 100644 --- a/packages/nx/src/command-line/import/import.ts +++ b/packages/nx/src/command-line/import/import.ts @@ -1,4 +1,8 @@ -import { join, relative, resolve } from 'path'; +import { dirname, join, relative, resolve } from 'path'; +import { minimatch } from 'minimatch'; +import { existsSync, promises as fsp } from 'node:fs'; +import * as chalk from 'chalk'; +import { load as yamlLoad } from '@zkochan/js-yaml'; import { cloneFromUpstream, GitRepository } from '../../utils/git-utils'; import { stat, mkdir, rm } from 'node:fs/promises'; import { tmpdir } from 'tmp'; @@ -11,6 +15,9 @@ import { workspaceRoot } from '../../utils/workspace-root'; import { detectPackageManager, getPackageManagerCommand, + isWorkspacesEnabled, + PackageManager, + PackageManagerCommands, } from '../../utils/package-manager'; import { resetWorkspaceContext } from '../../utils/workspace-context'; import { runInstall } from '../init/implementation/utils'; @@ -21,6 +28,7 @@ import { getPackagesInPackageManagerWorkspace, needsInstall, } from './utils/needs-install'; +import { readPackageJson } from '../../project-graph/file-utils'; const importRemoteName = '__tmp_nx_import__'; @@ -41,6 +49,10 @@ export interface ImportOptions { * The directory in the destination repo to import into */ destination: string; + /** + * The depth to clone the source repository (limit this for faster clone times) + */ + depth: number; verbose: boolean; interactive: boolean; @@ -90,7 +102,7 @@ export async function importHandler(options: ImportOptions) { const sourceRepoPath = join(tempImportDirectory, 'repo'); const spinner = createSpinner( - `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath}` + `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath} (Use --depth to limit commit history and speed up clone times)` ).start(); try { await rm(tempImportDirectory, { recursive: true }); @@ -101,6 +113,7 @@ export async function importHandler(options: ImportOptions) { try { sourceGitClient = await cloneFromUpstream(sourceRemoteUrl, sourceRepoPath, { originName: importRemoteName, + depth: options.depth, }); } catch (e) { spinner.fail(`Failed to clone ${sourceRemoteUrl} into ${sourceRepoPath}`); @@ -110,6 +123,9 @@ export async function importHandler(options: ImportOptions) { } spinner.succeed(`Cloned into ${sourceRepoPath}`); + // Detecting the package manager before preparing the source repo for import. + const sourcePackageManager = detectPackageManager(sourceGitClient.root); + if (!ref) { const branchChoices = await sourceGitClient.listBranches(); ref = ( @@ -149,6 +165,7 @@ export async function importHandler(options: ImportOptions) { name: 'destination', message: 'Where in this workspace should the code be imported into?', required: true, + initial: source ? source : undefined, }, ]) ).destination; @@ -157,6 +174,23 @@ export async function importHandler(options: ImportOptions) { const absSource = join(sourceRepoPath, source); const absDestination = join(process.cwd(), destination); + const destinationGitClient = new GitRepository(process.cwd()); + await assertDestinationEmpty(destinationGitClient, absDestination); + + const tempImportBranch = getTempImportBranch(ref); + await sourceGitClient.addFetchRemote(importRemoteName, ref); + await sourceGitClient.fetch(importRemoteName, ref); + spinner.succeed(`Fetched ${ref} from ${sourceRemoteUrl}`); + spinner.start( + `Checking out a temporary branch, ${tempImportBranch} based on ${ref}` + ); + await sourceGitClient.checkout(tempImportBranch, { + new: true, + base: `${importRemoteName}/${ref}`, + }); + + spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`); + try { await stat(absSource); } catch (e) { @@ -165,11 +199,6 @@ export async function importHandler(options: ImportOptions) { ); } - const destinationGitClient = new GitRepository(process.cwd()); - await assertDestinationEmpty(destinationGitClient, absDestination); - - const tempImportBranch = getTempImportBranch(ref); - const packageManager = detectPackageManager(workspaceRoot); const originalPackageWorkspaces = await getPackagesInPackageManagerWorkspace( @@ -186,8 +215,7 @@ export async function importHandler(options: ImportOptions) { source, relativeDestination, tempImportBranch, - sourceRemoteUrl, - importRemoteName + sourceRemoteUrl ); await createTemporaryRemote( @@ -220,22 +248,74 @@ export async function importHandler(options: ImportOptions) { options.interactive ); - if (plugins.length > 0) { - output.log({ title: 'Installing Plugins' }); - installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts); + if (packageManager !== sourcePackageManager) { + output.warn({ + title: `Mismatched package managers`, + bodyLines: [ + `The source repository is using a different package manager (${sourcePackageManager}) than this workspace (${packageManager}).`, + `This could lead to install issues due to discrepancies in "package.json" features.`, + ], + }); + } - await destinationGitClient.amendCommit(); + // If install fails, we should continue since the errors could be resolved later. + let installFailed = false; + if (plugins.length > 0) { + try { + output.log({ title: 'Installing Plugins' }); + installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts); + + await destinationGitClient.amendCommit(); + } catch (e) { + installFailed = true; + output.error({ + title: `Install failed: ${e.message || 'Unknown error'}`, + bodyLines: [e.stack], + }); + } } else if (await needsInstall(packageManager, originalPackageWorkspaces)) { - output.log({ - title: 'Installing dependencies for imported code', - }); + try { + output.log({ + title: 'Installing dependencies for imported code', + }); + + runInstall(workspaceRoot, getPackageManagerCommand(packageManager)); + + await destinationGitClient.amendCommit(); + } catch (e) { + installFailed = true; + output.error({ + title: `Install failed: ${e.message || 'Unknown error'}`, + bodyLines: [e.stack], + }); + } + } - runInstall(workspaceRoot, getPackageManagerCommand(packageManager)); + console.log(await destinationGitClient.showStat()); - await destinationGitClient.amendCommit(); + if (installFailed) { + const pmc = getPackageManagerCommand(packageManager); + output.warn({ + title: `The import was successful, but the install failed`, + bodyLines: [ + `You may need to run "${pmc.install}" manually to resolve the issue. The error is logged above.`, + ], + }); } - console.log(await destinationGitClient.showStat()); + await warnOnMissingWorkspacesEntry(packageManager, pmc, relativeDestination); + + // When only a subdirectory is imported, there might be devDependencies in the root package.json file + // that needs to be ported over as well. + if (ref) { + output.log({ + title: `Check root dependencies`, + bodyLines: [ + `"dependencies" and "devDependencies" are not imported from the source repository (${sourceRemoteUrl}).`, + `You may need to add some of those dependencies to this workspace in order to run tasks successfully.`, + ], + }); + } output.log({ title: `Merging these changes into ${getBaseRef(nxJson)}`, @@ -274,3 +354,77 @@ async function createTemporaryRemote( await destinationGitClient.addGitRemote(remoteName, sourceRemoteUrl); await destinationGitClient.fetch(remoteName); } + +// If the user imports a project that isn't in NPM/Yarn/PNPM workspaces, then its dependencies +// will not be installed. We should warn users and provide instructions on how to fix this. +async function warnOnMissingWorkspacesEntry( + pm: PackageManager, + pmc: PackageManagerCommands, + pkgPath: string +) { + if (!isWorkspacesEnabled(pm, workspaceRoot)) { + output.warn({ + title: `Missing workspaces in package.json`, + bodyLines: + pm === 'npm' + ? [ + `We recommend enabling NPM workspaces to install dependencies for the imported project.`, + `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`, + `See: https://docs.npmjs.com/cli/using-npm/workspaces`, + ] + : pm === 'yarn' + ? [ + `We recommend enabling Yarn workspaces to install dependencies for the imported project.`, + `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`, + `See: https://yarnpkg.com/features/workspaces`, + ] + : pm === 'bun' + ? [ + `We recommend enabling Bun workspaces to install dependencies for the imported project.`, + `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`, + `See: https://bun.sh/docs/install/workspaces`, + ] + : [ + `We recommend enabling PNPM workspaces to install dependencies for the imported project.`, + `Add the following entry to to pnpm-workspace.yaml and run "${pmc.install}":`, + chalk.bold(`packages:\n - '${pkgPath}'`), + `See: https://pnpm.io/workspaces`, + ], + }); + } else { + // Check if the new package is included in existing workspaces entries. If not, warn the user. + let workspaces: string[] | null = null; + + if (pm === 'npm' || pm === 'yarn' || pm === 'bun') { + const packageJson = readPackageJson(); + workspaces = packageJson.workspaces; + } else if (pm === 'pnpm') { + const yamlPath = join(workspaceRoot, 'pnpm-workspace.yaml'); + if (existsSync(yamlPath)) { + const yamlContent = await fsp.readFile(yamlPath, 'utf-8'); + const yaml = yamlLoad(yamlContent); + workspaces = yaml.packages; + } + } + + if (workspaces) { + const isPkgIncluded = workspaces.some((w) => minimatch(pkgPath, w)); + if (!isPkgIncluded) { + const pkgsDir = dirname(pkgPath); + output.warn({ + title: `Project missing in workspaces`, + bodyLines: + pm === 'npm' || pm === 'yarn' || pm === 'bun' + ? [ + `The imported project (${pkgPath}) is missing the "workspaces" field in package.json.`, + `Add "${pkgsDir}/*" to workspaces run "${pmc.install}".`, + ] + : [ + `The imported project (${pkgPath}) is missing the "packages" field in pnpm-workspaces.yaml.`, + `Add "${pkgsDir}/*" to packages run "${pmc.install}".`, + ], + }); + } + } + } +} diff --git a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts index 25cae28176531..160e3e280dfb7 100644 --- a/packages/nx/src/command-line/import/utils/prepare-source-repo.ts +++ b/packages/nx/src/command-line/import/utils/prepare-source-repo.ts @@ -1,6 +1,6 @@ import * as createSpinner from 'ora'; -import { basename, dirname, join, relative } from 'path'; -import { copyFile, mkdir, rm } from 'node:fs/promises'; +import { dirname, join, relative } from 'path'; +import { mkdir, rm } from 'node:fs/promises'; import { GitRepository } from '../../../utils/git-utils'; export async function prepareSourceRepo( @@ -9,124 +9,66 @@ export async function prepareSourceRepo( source: string, relativeDestination: string, tempImportBranch: string, - sourceRemoteUrl: string, - originName: string + sourceRemoteUrl: string ) { const spinner = createSpinner().start( `Fetching ${ref} from ${sourceRemoteUrl}` ); - await gitClient.addFetchRemote(originName, ref); - await gitClient.fetch(originName, ref); - spinner.succeed(`Fetched ${ref} from ${sourceRemoteUrl}`); - spinner.start( - `Checking out a temporary branch, ${tempImportBranch} based on ${ref}` - ); - await gitClient.checkout(tempImportBranch, { - new: true, - base: `${originName}/${ref}`, - }); - spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`); const relativeSourceDir = relative( gitClient.root, join(gitClient.root, source) ); - const destinationInSource = join(gitClient.root, relativeDestination); - spinner.start(`Moving files and git history to ${destinationInSource}`); - if (relativeSourceDir === '') { - const files = await gitClient.getGitFiles('.'); - try { - await rm(destinationInSource, { - recursive: true, - }); - } catch {} - await mkdir(destinationInSource, { recursive: true }); - const gitignores = new Set(); - for (const file of files) { - if (basename(file) === '.gitignore') { - gitignores.add(file); - continue; - } + if (relativeSourceDir !== '') { + if (await gitClient.hasFilterRepoInstalled()) { spinner.start( - `Moving files and git history to ${destinationInSource}: ${file}` + `Filtering git history to only include files in ${relativeSourceDir}` ); - - const newPath = join(destinationInSource, file); - - await mkdir(dirname(newPath), { recursive: true }); - try { - await gitClient.move(file, newPath); - } catch { - await wait(100); - await gitClient.move(file, newPath); - } + await gitClient.filterRepo(relativeSourceDir); + } else { + spinner.start( + `Filtering git history to only include files in ${relativeSourceDir} (this might take a few minutes -- install git-filter-repo for faster performance)` + ); + await gitClient.filterBranch(relativeSourceDir, tempImportBranch); } + spinner.succeed( + `Filtered git history to only include files in ${relativeSourceDir}` + ); + } + + const destinationInSource = join(gitClient.root, relativeDestination); + spinner.start(`Moving files and git history to ${destinationInSource}`); - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` + // The result of filter-branch will contain only the files in the subdirectory at its root. + const files = await gitClient.getGitFiles('.'); + try { + await rm(destinationInSource, { + recursive: true, + }); + } catch {} + await mkdir(destinationInSource, { recursive: true }); + for (const file of files) { + spinner.start( + `Moving files and git history to ${destinationInSource}: ${file}` ); - for (const gitignore of gitignores) { - await gitClient.move(gitignore, join(destinationInSource, gitignore)); - } - await gitClient.amendCommit(); - for (const gitignore of gitignores) { - await copyFile( - join(destinationInSource, gitignore), - join(gitClient.root, gitignore) - ); - } - } else { - let needsSquash = false; - const needsMove = destinationInSource !== join(gitClient.root, source); - if (needsMove) { - try { - await rm(destinationInSource, { - recursive: true, - }); - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` - ); - needsSquash = true; - } catch {} + const newPath = join(destinationInSource, file); - await mkdir(destinationInSource, { recursive: true }); + await mkdir(dirname(newPath), { recursive: true }); + try { + await gitClient.move(file, newPath); + } catch { + await wait(100); + await gitClient.move(file, newPath); } + } - const files = await gitClient.getGitFiles('.'); - for (const file of files) { - if (file === '.gitignore') { - continue; - } - spinner.start( - `Moving files and git history to ${destinationInSource}: ${file}` - ); + await gitClient.commit( + `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` + ); - if (!relative(source, file).startsWith('..')) { - if (needsMove) { - const newPath = join(destinationInSource, file); + await gitClient.amendCommit(); - await mkdir(dirname(newPath), { recursive: true }); - try { - await gitClient.move(file, newPath); - } catch { - await wait(100); - await gitClient.move(file, newPath); - } - } - } else { - await rm(join(gitClient.root, file), { - recursive: true, - }); - } - } - await gitClient.commit( - `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported` - ); - if (needsSquash) { - await gitClient.squashLastTwoCommits(); - } - } spinner.succeed( `${sourceRemoteUrl} has been prepared to be imported into this workspace on a temporary branch: ${tempImportBranch} in ${gitClient.root}` ); diff --git a/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap b/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap deleted file mode 100644 index 5dc1f3c4715d9..0000000000000 --- a/packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap +++ /dev/null @@ -1,37 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`git utils tests updateRebaseFile should squash the last 2 commits 1`] = ` -"pick 6a642190 chore(repo): hi -pick 022528d9 chore(repo): prepare for import -fixup 84ef7741 feat(repo): complete import of git@github.com:FrozenPandaz/created-vite-app.git - -# Rebase 3441f39e..84ef7741 onto 3441f39e (3 commands) -# -# Commands: -# p, pick = use commit -# r, reword = use commit, but edit the commit message -# e, edit = use commit, but stop for amending -# s, squash = use commit, but meld into previous commit -# f, fixup [-C | -c] = like "squash" but keep only the previous -# commit's log message, unless -C is used, in which case -# keep only this commit's message; -c is same as -C but -# opens the editor -# x, exec = run command (the rest of the line) using shell -# b, break = stop here (continue rebase later with 'git rebase --continue') -# d, drop = remove commit -# l, label